diff options
author | Anton Samokhvalov <pg83@yandex.ru> | 2022-02-10 16:45:15 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:15 +0300 |
commit | 72cb13b4aff9bc9cf22e49251bc8fd143f82538f (patch) | |
tree | da2c34829458c7d4e74bdfbdf85dff449e9e7fb8 /library/cpp | |
parent | 778e51ba091dc39e7b7fcab2b9cf4dbedfb6f2b5 (diff) | |
download | ydb-72cb13b4aff9bc9cf22e49251bc8fd143f82538f.tar.gz |
Restoring authorship annotation for Anton Samokhvalov <pg83@yandex.ru>. Commit 1 of 2.
Diffstat (limited to 'library/cpp')
1200 files changed, 61064 insertions, 61064 deletions
diff --git a/library/cpp/actors/core/actor.h b/library/cpp/actors/core/actor.h index ed29bd14b9..bc0df60536 100644 --- a/library/cpp/actors/core/actor.h +++ b/library/cpp/actors/core/actor.h @@ -16,7 +16,7 @@ namespace NActors { namespace NLog { struct TSettings; - } + } struct TActorContext; @@ -434,7 +434,7 @@ namespace NActors { }; -#define STFUNC_SIG TAutoPtr< ::NActors::IEventHandle>&ev, const ::NActors::TActorContext &ctx +#define STFUNC_SIG TAutoPtr< ::NActors::IEventHandle>&ev, const ::NActors::TActorContext &ctx #define STATEFN_SIG TAutoPtr<::NActors::IEventHandle>& ev #define STFUNC(funcName) void funcName(TAutoPtr< ::NActors::IEventHandle>& ev, const ::NActors::TActorContext& ctx) #define STATEFN(funcName) void funcName(TAutoPtr< ::NActors::IEventHandle>& ev, const ::NActors::TActorContext& ) diff --git a/library/cpp/actors/core/actor_bootstrapped.h b/library/cpp/actors/core/actor_bootstrapped.h index a37887c939..365a18f020 100644 --- a/library/cpp/actors/core/actor_bootstrapped.h +++ b/library/cpp/actors/core/actor_bootstrapped.h @@ -34,4 +34,4 @@ namespace NActors { : TActor<TDerived>(&TDerived::StateBootstrap) {} }; -} +} diff --git a/library/cpp/actors/core/actor_coroutine.cpp b/library/cpp/actors/core/actor_coroutine.cpp index 0ab4d2b24d..ef92fa03ae 100644 --- a/library/cpp/actors/core/actor_coroutine.cpp +++ b/library/cpp/actors/core/actor_coroutine.cpp @@ -162,4 +162,4 @@ namespace NActors { } } -} +} diff --git a/library/cpp/actors/core/actor_coroutine.h b/library/cpp/actors/core/actor_coroutine.h index 6bcb768eaf..dc17f5c9e4 100644 --- a/library/cpp/actors/core/actor_coroutine.h +++ b/library/cpp/actors/core/actor_coroutine.h @@ -171,4 +171,4 @@ namespace NActors { } }; -} +} diff --git a/library/cpp/actors/core/actor_coroutine_ut.cpp b/library/cpp/actors/core/actor_coroutine_ut.cpp index 951512b877..731b14f145 100644 --- a/library/cpp/actors/core/actor_coroutine_ut.cpp +++ b/library/cpp/actors/core/actor_coroutine_ut.cpp @@ -105,7 +105,7 @@ Y_UNIT_TEST_SUITE(ActorCoro) { THolder<TActorSystemSetup> setup = MakeHolder<TActorSystemSetup>(); setup->NodeId = 0; setup->ExecutorsCount = 1; - setup->Executors.Reset(new TAutoPtr<IExecutorPool>[setup->ExecutorsCount]); + setup->Executors.Reset(new TAutoPtr<IExecutorPool>[setup->ExecutorsCount]); for (ui32 i = 0; i < setup->ExecutorsCount; ++i) { setup->Executors[i] = new TBasicExecutorPool(i, 5, 10, "basic"); } diff --git a/library/cpp/actors/core/buffer.cpp b/library/cpp/actors/core/buffer.cpp index 48128d76ef..cb8aaeeadd 100644 --- a/library/cpp/actors/core/buffer.cpp +++ b/library/cpp/actors/core/buffer.cpp @@ -2,7 +2,7 @@ #include <util/system/yassert.h> -#include <algorithm> +#include <algorithm> TBufferBase::TBufferBase(size_t size) noexcept : Size(size) diff --git a/library/cpp/actors/core/buffer.h b/library/cpp/actors/core/buffer.h index 95425046d6..6229114f0b 100644 --- a/library/cpp/actors/core/buffer.h +++ b/library/cpp/actors/core/buffer.h @@ -1,6 +1,6 @@ #pragma once -#include <limits> +#include <limits> class TConstBuffer; class TMutableBuffer; diff --git a/library/cpp/actors/core/callstack.cpp b/library/cpp/actors/core/callstack.cpp index 9297c1a079..dffd3df72f 100644 --- a/library/cpp/actors/core/callstack.cpp +++ b/library/cpp/actors/core/callstack.cpp @@ -7,7 +7,7 @@ namespace NActors { namespace { void (*PreviousFormatBackTrace)(IOutputStream*) = 0; ui32 ActorBackTraceEnableCounter = 0; - } + } void ActorFormatBackTrace(IOutputStream* out) { TStringStream str; @@ -88,6 +88,6 @@ namespace NActors { str << Endl; } } -} +} #endif diff --git a/library/cpp/actors/core/callstack.h b/library/cpp/actors/core/callstack.h index 176717d2ae..0f57e6cb18 100644 --- a/library/cpp/actors/core/callstack.h +++ b/library/cpp/actors/core/callstack.h @@ -44,7 +44,7 @@ namespace NActors { void EnableActorCallstack(); void DisableActorCallstack(); -} +} #else @@ -53,6 +53,6 @@ namespace NActors { inline void DisableActorCallstack(){}; -} +} #endif diff --git a/library/cpp/actors/core/event.h b/library/cpp/actors/core/event.h index 6ff02aaf94..7291699913 100644 --- a/library/cpp/actors/core/event.h +++ b/library/cpp/actors/core/event.h @@ -57,8 +57,8 @@ namespace NActors { }; public: - template <typename TEv> - inline TEv* CastAsLocal() const noexcept { + template <typename TEv> + inline TEv* CastAsLocal() const noexcept { auto fits = GetTypeRewrite() == TEv::EventType; return fits ? static_cast<TEv*>(Event.Get()) : nullptr; diff --git a/library/cpp/actors/core/event_load.h b/library/cpp/actors/core/event_load.h index 0dab1dd374..b7b9bfbe05 100644 --- a/library/cpp/actors/core/event_load.h +++ b/library/cpp/actors/core/event_load.h @@ -36,9 +36,9 @@ namespace NActors { TEventSerializedData(const TEventSerializedData& original, TString extraBuffer) : Rope(original.Rope) , ExtendedFormat(original.ExtendedFormat) - { + { Append(std::move(extraBuffer)); - } + } TEventSerializedData(TString buffer, bool extendedFormat) : ExtendedFormat(extendedFormat) diff --git a/library/cpp/actors/core/event_local.h b/library/cpp/actors/core/event_local.h index 2845aa94dd..f337b39fbe 100644 --- a/library/cpp/actors/core/event_local.h +++ b/library/cpp/actors/core/event_local.h @@ -47,7 +47,7 @@ namespace NActors { }; template <typename TEv, ui32 TEventType> - class TEventSimple: public TEventBase<TEv, TEventType> { + class TEventSimple: public TEventBase<TEv, TEventType> { public: TString ToStringHeader() const override { static TString header(TypeName<TEv>()); diff --git a/library/cpp/actors/core/event_pb.cpp b/library/cpp/actors/core/event_pb.cpp index 018ff9ac34..9ba5014755 100644 --- a/library/cpp/actors/core/event_pb.cpp +++ b/library/cpp/actors/core/event_pb.cpp @@ -220,4 +220,4 @@ namespace NActors { Buffers->Append(*s); return true; } -} +} diff --git a/library/cpp/actors/core/event_pb.h b/library/cpp/actors/core/event_pb.h index d7546b901a..61f3916dce 100644 --- a/library/cpp/actors/core/event_pb.h +++ b/library/cpp/actors/core/event_pb.h @@ -141,14 +141,14 @@ namespace NActors { TEventPBBase() = default; explicit TEventPBBase(const TRecord& rec) - { + { Record = rec; - } + } explicit TEventPBBase(TRecord&& rec) - { + { Record = std::move(rec); - } + } TString ToStringHeader() const override { return Record.GetTypeName(); @@ -394,7 +394,7 @@ namespace NActors { {} }; - template <typename TEv, typename TRecord, ui32 TEventType> + template <typename TEv, typename TRecord, ui32 TEventType> class TEventPB : public TEventPBBase<TEv, TRecord, TEventType, TRecordHolder<TRecord> > { typedef TEventPBBase<TEv, TRecord, TEventType, TRecordHolder<TRecord> > TPbBase; // NOTE: No extra fields allowed: TEventPB must be a "template typedef" @@ -406,25 +406,25 @@ namespace NActors { using TEventPBWithArena = TEventPBBase<TEv, TRecord, TEventType, TArenaRecordHolder<TRecord, InitialBlockSize, MaxBlockSize> >; template <typename TEv, typename TRecord, ui32 TEventType> - class TEventShortDebugPB: public TEventPB<TEv, TRecord, TEventType> { - public: - using TBase = TEventPB<TEv, TRecord, TEventType>; - TEventShortDebugPB() = default; - explicit TEventShortDebugPB(const TRecord& rec) - : TBase(rec) - { - } - explicit TEventShortDebugPB(TRecord&& rec) - : TBase(std::move(rec)) - { - } - TString ToString() const override { - return TypeName<TEv>() + " { " + TBase::Record.ShortDebugString() + " }"; - } - }; + class TEventShortDebugPB: public TEventPB<TEv, TRecord, TEventType> { + public: + using TBase = TEventPB<TEv, TRecord, TEventType>; + TEventShortDebugPB() = default; + explicit TEventShortDebugPB(const TRecord& rec) + : TBase(rec) + { + } + explicit TEventShortDebugPB(TRecord&& rec) + : TBase(std::move(rec)) + { + } + TString ToString() const override { + return TypeName<TEv>() + " { " + TBase::Record.ShortDebugString() + " }"; + } + }; template <typename TEv, typename TRecord, ui32 TEventType> - class TEventPreSerializedPB: public TEventPB<TEv, TRecord, TEventType> { + class TEventPreSerializedPB: public TEventPB<TEv, TRecord, TEventType> { protected: using TBase = TEventPB<TEv, TRecord, TEventType>; using TSelf = TEventPreSerializedPB<TEv, TRecord, TEventType>; @@ -437,13 +437,13 @@ namespace NActors { explicit TEventPreSerializedPB(const TRecord& rec) : TBase(rec) - { - } + { + } explicit TEventPreSerializedPB(TRecord&& rec) : TBase(std::move(rec)) - { - } + { + } // when remote event received locally this method will merge preserialized data const TRecord& GetRecord() { diff --git a/library/cpp/actors/core/event_pb_ut.cpp b/library/cpp/actors/core/event_pb_ut.cpp index a16c3092b3..c0f3ec5e8d 100644 --- a/library/cpp/actors/core/event_pb_ut.cpp +++ b/library/cpp/actors/core/event_pb_ut.cpp @@ -10,18 +10,18 @@ Y_UNIT_TEST_SUITE(TEventSerialization) { SerializeToArcadiaStream(NActors::TChunkSerializer* chunker) const override { return msg->SerializeToZeroCopyStream(chunker); } - bool IsSerializable() const override { - return true; - } + bool IsSerializable() const override { + return true; + } TString ToStringHeader() const override { - return TString(); - } - virtual TString Serialize() const { - return TString(); - } + return TString(); + } + virtual TString Serialize() const { + return TString(); + } ui32 Type() const override { - return 0; - }; + return 0; + }; }; Y_UNIT_TEST(Coroutine) { diff --git a/library/cpp/actors/core/interconnect.h b/library/cpp/actors/core/interconnect.h index 679a4b8cc6..10c036fcee 100644 --- a/library/cpp/actors/core/interconnect.h +++ b/library/cpp/actors/core/interconnect.h @@ -119,7 +119,7 @@ namespace NActors { enum EEv { EvForward = EventSpaceBegin(TEvents::ES_INTERCONNECT), EvResolveNode, // resolve info about node (internal) - EvNodeAddress, // node info (internal) + EvNodeAddress, // node info (internal) EvConnectNode, // request proxy to establish connection (like: we would send something there soon) EvAcceptIncoming, EvNodeConnected, // node connected notify @@ -244,9 +244,9 @@ namespace NActors { THolder<TNodeInfo> Node; }; - struct TEvClosePeerSocket : TEventLocal<TEvClosePeerSocket, EvClosePeerSocket> {}; + struct TEvClosePeerSocket : TEventLocal<TEvClosePeerSocket, EvClosePeerSocket> {}; - struct TEvCloseInputSession : TEventLocal<TEvCloseInputSession, EvCloseInputSession> {}; + struct TEvCloseInputSession : TEventLocal<TEvCloseInputSession, EvCloseInputSession> {}; struct TEvPoisonSession : TEventLocal<TEvPoisonSession, EvPoisonSession> {}; diff --git a/library/cpp/actors/core/log.cpp b/library/cpp/actors/core/log.cpp index 5f63b5af58..4efdc01ad0 100644 --- a/library/cpp/actors/core/log.cpp +++ b/library/cpp/actors/core/log.cpp @@ -3,14 +3,14 @@ #include <library/cpp/monlib/service/pages/templates.h> -static_assert(int(NActors::NLog::PRI_EMERG) == int(::TLOG_EMERG), "expect int(NActors::NLog::PRI_EMERG) == int(::TLOG_EMERG)"); -static_assert(int(NActors::NLog::PRI_ALERT) == int(::TLOG_ALERT), "expect int(NActors::NLog::PRI_ALERT) == int(::TLOG_ALERT)"); -static_assert(int(NActors::NLog::PRI_CRIT) == int(::TLOG_CRIT), "expect int(NActors::NLog::PRI_CRIT) == int(::TLOG_CRIT)"); -static_assert(int(NActors::NLog::PRI_ERROR) == int(::TLOG_ERR), "expect int(NActors::NLog::PRI_ERROR) == int(::TLOG_ERR)"); -static_assert(int(NActors::NLog::PRI_WARN) == int(::TLOG_WARNING), "expect int(NActors::NLog::PRI_WARN) == int(::TLOG_WARNING)"); -static_assert(int(NActors::NLog::PRI_NOTICE) == int(::TLOG_NOTICE), "expect int(NActors::NLog::PRI_NOTICE) == int(::TLOG_NOTICE)"); -static_assert(int(NActors::NLog::PRI_INFO) == int(::TLOG_INFO), "expect int(NActors::NLog::PRI_INFO) == int(::TLOG_INFO)"); -static_assert(int(NActors::NLog::PRI_DEBUG) == int(::TLOG_DEBUG), "expect int(NActors::NLog::PRI_DEBUG) == int(::TLOG_DEBUG)"); +static_assert(int(NActors::NLog::PRI_EMERG) == int(::TLOG_EMERG), "expect int(NActors::NLog::PRI_EMERG) == int(::TLOG_EMERG)"); +static_assert(int(NActors::NLog::PRI_ALERT) == int(::TLOG_ALERT), "expect int(NActors::NLog::PRI_ALERT) == int(::TLOG_ALERT)"); +static_assert(int(NActors::NLog::PRI_CRIT) == int(::TLOG_CRIT), "expect int(NActors::NLog::PRI_CRIT) == int(::TLOG_CRIT)"); +static_assert(int(NActors::NLog::PRI_ERROR) == int(::TLOG_ERR), "expect int(NActors::NLog::PRI_ERROR) == int(::TLOG_ERR)"); +static_assert(int(NActors::NLog::PRI_WARN) == int(::TLOG_WARNING), "expect int(NActors::NLog::PRI_WARN) == int(::TLOG_WARNING)"); +static_assert(int(NActors::NLog::PRI_NOTICE) == int(::TLOG_NOTICE), "expect int(NActors::NLog::PRI_NOTICE) == int(::TLOG_NOTICE)"); +static_assert(int(NActors::NLog::PRI_INFO) == int(::TLOG_INFO), "expect int(NActors::NLog::PRI_INFO) == int(::TLOG_INFO)"); +static_assert(int(NActors::NLog::PRI_DEBUG) == int(::TLOG_DEBUG), "expect int(NActors::NLog::PRI_DEBUG) == int(::TLOG_DEBUG)"); static_assert(int(NActors::NLog::PRI_TRACE) == int(::TLOG_RESOURCES), "expect int(NActors::NLog::PRI_TRACE) == int(::TLOG_RESOURCES)"); namespace { @@ -706,8 +706,8 @@ namespace NActors { public: TLineFileLogBackend(const TString& path) : TFileLogBackend(path) - { - } + { + } // Append newline after every record void WriteData(const TLogRecord& rec) override { @@ -750,4 +750,4 @@ namespace NActors { TAutoPtr<TLogBackend> CreateCompositeLogBackend(TVector<TAutoPtr<TLogBackend>>&& underlyingBackends) { return new TCompositeLogBackend(std::move(underlyingBackends)); } -} +} diff --git a/library/cpp/actors/core/log.h b/library/cpp/actors/core/log.h index c11a7cf3c1..6b66977fa1 100644 --- a/library/cpp/actors/core/log.h +++ b/library/cpp/actors/core/log.h @@ -366,4 +366,4 @@ namespace NActors { MemLogWrite(str.data(), str.size(), true); DeliverLogMessage(actorCtxOrSystem, mPriority, mComponent, std::move(str)); } -} +} diff --git a/library/cpp/actors/core/log_iface.h b/library/cpp/actors/core/log_iface.h index b331db9ca8..3acdcd69bc 100644 --- a/library/cpp/actors/core/log_iface.h +++ b/library/cpp/actors/core/log_iface.h @@ -4,55 +4,55 @@ #include "event_local.h" namespace NActors { - namespace NLog { - using EComponent = int; - - enum EPriority : ui16 { // migrate it to EPrio whenever possible - PRI_EMERG, - PRI_ALERT, - PRI_CRIT, - PRI_ERROR, - PRI_WARN, - PRI_NOTICE, - PRI_INFO, - PRI_DEBUG, - PRI_TRACE - }; - - enum class EPrio : ui16 { - Emerg = 0, - Alert = 1, - Crit = 2, - Error = 3, - Warn = 4, - Notice = 5, - Info = 6, - Debug = 7, - Trace = 8, - }; - - struct TLevel { - TLevel(ui32 raw) - : Raw(raw) - { - } - - TLevel(EPrio prio) - : Raw((ui16(prio) + 1) << 8) - { - } - - EPrio ToPrio() const noexcept { - const auto major = Raw >> 8; - - return major > 0 ? EPrio(major - 1) : EPrio::Emerg; - } - - bool IsUrgentAbortion() const noexcept { - return (Raw >> 8) == 0; - } - - /* Generalized monotonic level value composed with major and minor + namespace NLog { + using EComponent = int; + + enum EPriority : ui16 { // migrate it to EPrio whenever possible + PRI_EMERG, + PRI_ALERT, + PRI_CRIT, + PRI_ERROR, + PRI_WARN, + PRI_NOTICE, + PRI_INFO, + PRI_DEBUG, + PRI_TRACE + }; + + enum class EPrio : ui16 { + Emerg = 0, + Alert = 1, + Crit = 2, + Error = 3, + Warn = 4, + Notice = 5, + Info = 6, + Debug = 7, + Trace = 8, + }; + + struct TLevel { + TLevel(ui32 raw) + : Raw(raw) + { + } + + TLevel(EPrio prio) + : Raw((ui16(prio) + 1) << 8) + { + } + + EPrio ToPrio() const noexcept { + const auto major = Raw >> 8; + + return major > 0 ? EPrio(major - 1) : EPrio::Emerg; + } + + bool IsUrgentAbortion() const noexcept { + return (Raw >> 8) == 0; + } + + /* Generalized monotonic level value composed with major and minor levels. Minor is used for verbosity within major, basic EPrio mapped to (EPrio + 1, 0) and Major = 0 is reserved as special space with meaning like EPrio::Emerg but with extened actions. @@ -60,25 +60,25 @@ namespace NActors { idea how to handle special emergency actions. */ - ui32 Raw = 0; // ((ui16(EPrio) + 1) << 8) | ui8(minor) - }; + ui32 Raw = 0; // ((ui16(EPrio) + 1) << 8) | ui8(minor) + }; - enum class EEv { - Log = EventSpaceBegin(TEvents::ES_LOGGER), - LevelReq, - LevelResp, - Ignored, - End - }; + enum class EEv { + Log = EventSpaceBegin(TEvents::ES_LOGGER), + LevelReq, + LevelResp, + Ignored, + End + }; - static_assert(int(EEv::End) < EventSpaceEnd(TEvents::ES_LOGGER), ""); + static_assert(int(EEv::End) < EventSpaceEnd(TEvents::ES_LOGGER), ""); - class TEvLog: public TEventLocal<TEvLog, int(EEv::Log)> { - public: + class TEvLog: public TEventLocal<TEvLog, int(EEv::Log)> { + public: TEvLog(TInstant stamp, TLevel level, EComponent comp, const TString &line) - : Stamp(stamp) - , Level(level) - , Component(comp) + : Stamp(stamp) + , Level(level) + , Component(comp) , Line(line) { } @@ -87,23 +87,23 @@ namespace NActors { : Stamp(stamp) , Level(level) , Component(comp) - , Line(std::move(line)) - { - } - - TEvLog(EPriority prio, EComponent comp, TString line, TInstant time = TInstant::Now()) - : Stamp(time) - , Level(EPrio(prio)) - , Component(comp) - , Line(std::move(line)) - { - } - - const TInstant Stamp = TInstant::Max(); - const TLevel Level; - const EComponent Component = 0; - TString Line; - }; - - } + , Line(std::move(line)) + { + } + + TEvLog(EPriority prio, EComponent comp, TString line, TInstant time = TInstant::Now()) + : Stamp(time) + , Level(EPrio(prio)) + , Component(comp) + , Line(std::move(line)) + { + } + + const TInstant Stamp = TInstant::Max(); + const TLevel Level; + const EComponent Component = 0; + TString Line; + }; + + } } diff --git a/library/cpp/actors/core/log_settings.cpp b/library/cpp/actors/core/log_settings.cpp index f52f2fc5d2..c4e5a39d46 100644 --- a/library/cpp/actors/core/log_settings.cpp +++ b/library/cpp/actors/core/log_settings.cpp @@ -225,6 +225,6 @@ namespace NActors { return InvalidComponent; } - } + } -} +} diff --git a/library/cpp/actors/core/log_settings.h b/library/cpp/actors/core/log_settings.h index 7fe4504edd..9b611e4285 100644 --- a/library/cpp/actors/core/log_settings.h +++ b/library/cpp/actors/core/log_settings.h @@ -41,8 +41,8 @@ namespace NActors { static const int InvalidComponent = -1; // Functions converts EComponent id to string - using EComponentToStringFunc = std::function<const TString&(EComponent)>; - ; + using EComponentToStringFunc = std::function<const TString&(EComponent)>; + ; // Log settings struct TComponentSettings { @@ -109,7 +109,7 @@ namespace NActors { void Append(EComponent minVal, EComponent maxVal, EComponentToStringFunc func); - template <typename T> + template <typename T> void Append(T minVal, T maxVal, const TString& (*func)(T)) { Append( static_cast<EComponent>(minVal), @@ -171,6 +171,6 @@ namespace NActors { EPriority priority, EComponent component, TString& explanation); }; - } + } -} +} diff --git a/library/cpp/actors/core/mon.h b/library/cpp/actors/core/mon.h index c450f2338e..e8d3ab753d 100644 --- a/library/cpp/actors/core/mon.h +++ b/library/cpp/actors/core/mon.h @@ -17,7 +17,7 @@ namespace NActors { End }; - static_assert(End < EventSpaceEnd(NActors::TEvents::ES_MON), "expect End < EventSpaceEnd(NActors::TEvents::ES_MON)"); + static_assert(End < EventSpaceEnd(NActors::TEvents::ES_MON), "expect End < EventSpaceEnd(NActors::TEvents::ES_MON)"); // request info from an actor in HTML format struct TEvHttpInfo: public NActors::TEventLocal<TEvHttpInfo, HttpInfo> { @@ -229,6 +229,6 @@ namespace NActors { } }; - } + } -} +} diff --git a/library/cpp/actors/core/mon_stats.h b/library/cpp/actors/core/mon_stats.h index d55552af0c..7009c4ddfb 100644 --- a/library/cpp/actors/core/mon_stats.h +++ b/library/cpp/actors/core/mon_stats.h @@ -89,8 +89,8 @@ namespace NActors { TExecutorThreadStats(size_t activityVecSize = 1) // must be not empty as 0 used as default : ElapsedTicksByActivity(activityVecSize) - , ReceivedEventsByActivity(activityVecSize) - , ActorsAliveByActivity(activityVecSize) + , ReceivedEventsByActivity(activityVecSize) + , ActorsAliveByActivity(activityVecSize) , ScheduledEventsByActivity(activityVecSize) {} @@ -144,4 +144,4 @@ namespace NActors { } }; -} +} diff --git a/library/cpp/actors/core/probes.h b/library/cpp/actors/core/probes.h index 4912d6dd26..052be33906 100644 --- a/library/cpp/actors/core/probes.h +++ b/library/cpp/actors/core/probes.h @@ -74,7 +74,7 @@ PROBE(ActorsystemScheduler, GROUPS("Durations"), \ TYPES(ui64, ui64, ui32, ui32, ui64, ui64), \ NAMES("timeUs", "timerfd_expirations", "eventsGottenFromQueues", "eventsSent", \ - "eventsInSendQueue", "eventSchedulingErrorUs")) \ + "eventsInSendQueue", "eventSchedulingErrorUs")) \ PROBE(ForwardEvent, GROUPS("Orbit", "InterconnectSessionTCP"), \ TYPES(ui32, ui32, ui32, LWTYPE_ACTORID, LWTYPE_ACTORID, ui64, ui32), \ NAMES("peerId", "type", "flags", LWNAME_ACTORID("r"), LWNAME_ACTORID("s"), \ @@ -166,7 +166,7 @@ PROBE(MoveCpu, GROUPS("PoolCpuBalancer"), \ TYPES(ui32, ui64, TString, TString, ui32), \ NAMES("fromPoolId", "toPoolId", "fromPool", "toPool", "cpu")) \ - /**/ + /**/ LWTRACE_DECLARE_PROVIDER(ACTORLIB_PROVIDER) diff --git a/library/cpp/actors/core/process_stats.cpp b/library/cpp/actors/core/process_stats.cpp index 0e1dbd0031..8876ab3f34 100644 --- a/library/cpp/actors/core/process_stats.cpp +++ b/library/cpp/actors/core/process_stats.cpp @@ -300,4 +300,4 @@ namespace { IActor* CreateProcStatCollector(TDuration interval, NMonitoring::TMetricRegistry& registry) { return new TRegistryCollector(interval, registry); } -} +} diff --git a/library/cpp/actors/core/process_stats.h b/library/cpp/actors/core/process_stats.h index 66346d0b5a..707ccc9ffa 100644 --- a/library/cpp/actors/core/process_stats.h +++ b/library/cpp/actors/core/process_stats.h @@ -63,4 +63,4 @@ namespace NActors { IActor* CreateProcStatCollector(ui32 intervalSec, NMonitoring::TDynamicCounterPtr counters); IActor* CreateProcStatCollector(TDuration interval, NMonitoring::TMetricRegistry& registry); -} +} diff --git a/library/cpp/actors/core/scheduler_actor.cpp b/library/cpp/actors/core/scheduler_actor.cpp index febc5e40dd..484678a349 100644 --- a/library/cpp/actors/core/scheduler_actor.cpp +++ b/library/cpp/actors/core/scheduler_actor.cpp @@ -8,15 +8,15 @@ #include <util/system/hp_timer.h> #ifdef __linux__ -#include <sys/timerfd.h> -#include <errno.h> +#include <sys/timerfd.h> +#include <errno.h> LWTRACE_USING(ACTORLIB_PROVIDER); namespace NActors { - class TTimerDescriptor: public TSharedDescriptor { + class TTimerDescriptor: public TSharedDescriptor { const int Descriptor; - + public: TTimerDescriptor() : Descriptor(timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK)) @@ -33,7 +33,7 @@ namespace NActors { } }; - class TSchedulerActor: public TActor<TSchedulerActor> { + class TSchedulerActor: public TActor<TSchedulerActor> { const TSchedulerConfig Cfg; TIntrusivePtr<TSharedDescriptor> TimerDescriptor; @@ -47,7 +47,7 @@ namespace NActors { ui64 ActiveTick; typedef TMap<ui64, TAutoPtr<NSchedulerQueue::TQueueType>> TMomentMap; // intrasecond queues - typedef THashMap<ui64, TAutoPtr<TMomentMap>> TScheduleMap; // over-second schedule + typedef THashMap<ui64, TAutoPtr<TMomentMap>> TScheduleMap; // over-second schedule TScheduleMap ScheduleMap; @@ -55,7 +55,7 @@ namespace NActors { static const ui64 IntrasecondThreshold = 1048576; // ~second TAutoPtr<TMomentMap> ActiveSec; - volatile ui64* CurrentTimestamp = nullptr; + volatile ui64* CurrentTimestamp = nullptr; volatile ui64* CurrentMonotonic = nullptr; TDeque<TAutoPtr<IEventHandle>> EventsToBeSent; @@ -64,7 +64,7 @@ namespace NActors { return IActor::ACTOR_SYSTEM_SCHEDULER_ACTOR; } - TSchedulerActor(const TSchedulerConfig& cfg) + TSchedulerActor(const TSchedulerConfig& cfg) : TActor(&TSchedulerActor::StateFunc) , Cfg(cfg) , TimerDescriptor(new TTimerDescriptor()) @@ -75,8 +75,8 @@ namespace NActors { Become(&TSchedulerActor::StateFunc); } - void Handle(TEvSchedulerInitialize::TPtr& ev, const TActorContext& ctx) { - const TEvSchedulerInitialize& evInitialize = *ev->Get(); + void Handle(TEvSchedulerInitialize::TPtr& ev, const TActorContext& ctx) { + const TEvSchedulerInitialize& evInitialize = *ev->Get(); Y_ASSERT(evInitialize.ScheduleReaders.size() != 0); Readers.resize(evInitialize.ScheduleReaders.size()); Copy(evInitialize.ScheduleReaders.begin(), evInitialize.ScheduleReaders.end(), Readers.begin()); @@ -114,7 +114,7 @@ namespace NActors { AtomicStore(CurrentMonotonic, MonotonicTime); } - void TryUpdateTime(NHPTimer::STime* lastTimeUpdate) { + void TryUpdateTime(NHPTimer::STime* lastTimeUpdate) { NHPTimer::STime hpnow; GetTimeFast(&hpnow); const ui64 elapsedCycles = hpnow > *lastTimeUpdate ? hpnow - *lastTimeUpdate : 0; @@ -256,7 +256,7 @@ namespace NActors { ) }; - IActor* CreateSchedulerActor(const TSchedulerConfig& cfg) { + IActor* CreateSchedulerActor(const TSchedulerConfig& cfg) { if (cfg.UseSchedulerActor) { return new TSchedulerActor(cfg); } else { @@ -264,16 +264,16 @@ namespace NActors { } } -} +} #else // linux namespace NActors { - IActor* CreateSchedulerActor(const TSchedulerConfig& cfg) { + IActor* CreateSchedulerActor(const TSchedulerConfig& cfg) { Y_UNUSED(cfg); return nullptr; } -} +} #endif // linux diff --git a/library/cpp/actors/core/scheduler_actor.h b/library/cpp/actors/core/scheduler_actor.h index c2c561b43d..f743ed8d83 100644 --- a/library/cpp/actors/core/scheduler_actor.h +++ b/library/cpp/actors/core/scheduler_actor.h @@ -8,22 +8,22 @@ namespace NActors { struct TEvSchedulerInitialize : TEventLocal<TEvSchedulerInitialize, TEvents::TSystem::Bootstrap> { TVector<NSchedulerQueue::TReader*> ScheduleReaders; - volatile ui64* CurrentTimestamp; + volatile ui64* CurrentTimestamp; volatile ui64* CurrentMonotonic; TEvSchedulerInitialize(const TVector<NSchedulerQueue::TReader*>& scheduleReaders, volatile ui64* currentTimestamp, volatile ui64* currentMonotonic) : ScheduleReaders(scheduleReaders) , CurrentTimestamp(currentTimestamp) , CurrentMonotonic(currentMonotonic) - { - } + { + } }; - IActor* CreateSchedulerActor(const TSchedulerConfig& cfg); + IActor* CreateSchedulerActor(const TSchedulerConfig& cfg); inline TActorId MakeSchedulerActorId() { char x[12] = {'s', 'c', 'h', 'e', 'd', 'u', 'l', 'e', 'r', 's', 'e', 'r'}; return TActorId(0, TStringBuf(x, 12)); } -} +} diff --git a/library/cpp/actors/core/scheduler_actor_ut.cpp b/library/cpp/actors/core/scheduler_actor_ut.cpp index 09b7369d36..b449494e6c 100644 --- a/library/cpp/actors/core/scheduler_actor_ut.cpp +++ b/library/cpp/actors/core/scheduler_actor_ut.cpp @@ -30,13 +30,13 @@ Y_UNIT_TEST_SUITE(SchedulerActor) { { } - void Bootstrap(const TActorContext& ctx) { + void Bootstrap(const TActorContext& ctx) { LastWakeup = ctx.Now(); Become(&TThis::StateFunc); ctx.Schedule(ScheduleDelta, new TEvents::TEvWakeup()); } - void Handle(TEvents::TEvWakeup::TPtr& /*ev*/, const TActorContext& ctx) { + void Handle(TEvents::TEvWakeup::TPtr& /*ev*/, const TActorContext& ctx) { const TInstant now = ctx.Now(); UNIT_ASSERT(now - LastWakeup >= ScheduleDelta); LastWakeup = now; @@ -48,25 +48,25 @@ Y_UNIT_TEST_SUITE(SchedulerActor) { } } - STRICT_STFUNC(StateFunc, {HFunc(TEvents::TEvWakeup, Handle)}) + STRICT_STFUNC(StateFunc, {HFunc(TEvents::TEvWakeup, Handle)}) }; void Test(TAtomicBase eventsTotalCount, ui32 scheduleDeltaMs) { THolder<TActorSystemSetup> setup = MakeHolder<TActorSystemSetup>(); setup->NodeId = 0; setup->ExecutorsCount = 1; - setup->Executors.Reset(new TAutoPtr<IExecutorPool>[setup->ExecutorsCount]); + setup->Executors.Reset(new TAutoPtr<IExecutorPool>[setup->ExecutorsCount]); for (ui32 i = 0; i < setup->ExecutorsCount; ++i) { setup->Executors[i] = new TBasicExecutorPool(i, 5, 10, "basic"); } // create poller actor (whether platform supports it) TActorId pollerActorId; - if (IActor* poller = CreatePollerActor()) { + if (IActor* poller = CreatePollerActor()) { pollerActorId = MakePollerActorId(); setup->LocalServices.emplace_back(pollerActorId, TActorSetupCmd(poller, TMailboxType::ReadAsFilled, 0)); } TActorId schedulerActorId; - if (IActor* schedulerActor = CreateSchedulerActor(TSchedulerConfig())) { + if (IActor* schedulerActor = CreateSchedulerActor(TSchedulerConfig())) { schedulerActorId = MakeSchedulerActorId(); setup->LocalServices.emplace_back(schedulerActorId, TActorSetupCmd(schedulerActor, TMailboxType::ReadAsFilled, 0)); } diff --git a/library/cpp/actors/core/scheduler_basic.cpp b/library/cpp/actors/core/scheduler_basic.cpp index fba200e16b..f9a486dc1b 100644 --- a/library/cpp/actors/core/scheduler_basic.cpp +++ b/library/cpp/actors/core/scheduler_basic.cpp @@ -248,7 +248,7 @@ namespace NActors { MainCycle.Destroy(); } -} +} #ifdef __linux__ @@ -261,12 +261,12 @@ namespace NActors { } } -} +} #else // __linux__ namespace NActors { - ISchedulerThread* CreateSchedulerThread(const TSchedulerConfig& config) { + ISchedulerThread* CreateSchedulerThread(const TSchedulerConfig& config) { return new TBasicSchedulerThread(config); } } diff --git a/library/cpp/actors/core/scheduler_basic.h b/library/cpp/actors/core/scheduler_basic.h index 2ccde39235..0237c3c0a8 100644 --- a/library/cpp/actors/core/scheduler_basic.h +++ b/library/cpp/actors/core/scheduler_basic.h @@ -27,7 +27,7 @@ namespace NActors { volatile bool StopFlag; typedef TMap<ui64, TAutoPtr<NSchedulerQueue::TQueueType>> TMomentMap; // intrasecond queues - typedef THashMap<ui64, TAutoPtr<TMomentMap>> TScheduleMap; // over-second schedule + typedef THashMap<ui64, TAutoPtr<TMomentMap>> TScheduleMap; // over-second schedule TScheduleMap ScheduleMap; @@ -50,7 +50,7 @@ namespace NActors { void Stop() override; }; - class TMockSchedulerThread: public ISchedulerThread { + class TMockSchedulerThread: public ISchedulerThread { public: virtual ~TMockSchedulerThread() override { } @@ -76,6 +76,6 @@ namespace NActors { } }; - ISchedulerThread* CreateSchedulerThread(const TSchedulerConfig& cfg); + ISchedulerThread* CreateSchedulerThread(const TSchedulerConfig& cfg); } diff --git a/library/cpp/actors/dnscachelib/dnscache.cpp b/library/cpp/actors/dnscachelib/dnscache.cpp index 649339ddb2..31eb8468eb 100644 --- a/library/cpp/actors/dnscachelib/dnscache.cpp +++ b/library/cpp/actors/dnscachelib/dnscache.cpp @@ -6,22 +6,22 @@ #include <util/system/guard.h> #include <util/datetime/systime.h> -const TDnsCache::THost TDnsCache::NullHost; +const TDnsCache::THost TDnsCache::NullHost; LWTRACE_USING(DNSCACHELIB_PROVIDER); -static_assert(sizeof(ares_channel) == sizeof(void*), "expect sizeof(ares_channel) == sizeof(void *)"); +static_assert(sizeof(ares_channel) == sizeof(void*), "expect sizeof(ares_channel) == sizeof(void *)"); TDnsCache::TDnsCache(bool allowIpv4, bool allowIpv6, time_t lifetime, time_t neg, ui32 timeout) - : EntryLifetime(lifetime) - , NegativeLifetime(neg) - , Timeout(TDuration::MicroSeconds(timeout)) - , AllowIpV4(allowIpv4) - , AllowIpV6(allowIpv6) - , ACacheHits(0) - , ACacheMisses(0) - , PtrCacheHits(0) - , PtrCacheMisses(0) + : EntryLifetime(lifetime) + , NegativeLifetime(neg) + , Timeout(TDuration::MicroSeconds(timeout)) + , AllowIpV4(allowIpv4) + , AllowIpV6(allowIpv6) + , ACacheHits(0) + , ACacheMisses(0) + , PtrCacheHits(0) + , PtrCacheMisses(0) { #ifdef _win_ if (ares_library_init(ARES_LIB_INIT_WIN32) != ARES_SUCCESS) { @@ -40,7 +40,7 @@ TDnsCache::TDnsCache(bool allowIpv4, bool allowIpv6, time_t lifetime, time_t neg LWPROBE(Created); } -TDnsCache::~TDnsCache(void) { +TDnsCache::~TDnsCache(void) { ares_channel chan = static_cast<ares_channel>(Channel); ares_cancel(chan); @@ -52,29 +52,29 @@ TDnsCache::~TDnsCache(void) { #endif } -TString TDnsCache::GetHostByAddr(const NAddr::IRemoteAddr& addr) { +TString TDnsCache::GetHostByAddr(const NAddr::IRemoteAddr& addr) { in6_addr key; if (addr.Addr()->sa_family == AF_INET6) { - const struct sockaddr_in6* s6 = (const struct sockaddr_in6*)(addr.Addr()); + const struct sockaddr_in6* s6 = (const struct sockaddr_in6*)(addr.Addr()); memcpy(&key, &s6->sin6_addr, sizeof(s6->sin6_addr)); } else if (addr.Addr()->sa_family == AF_INET) { - const struct sockaddr_in* s4 = (const struct sockaddr_in*)(addr.Addr()); + const struct sockaddr_in* s4 = (const struct sockaddr_in*)(addr.Addr()); memset(&key, 0, sizeof(key)); memcpy(&key, &s4->sin_addr, sizeof(s4->sin_addr)); } else { return ""; } - const TAddr& host = ResolveAddr(key, addr.Addr()->sa_family); + const TAddr& host = ResolveAddr(key, addr.Addr()->sa_family); return host.Hostname; } -TIpHost TDnsCache::Get(const TString& hostname) { +TIpHost TDnsCache::Get(const TString& hostname) { if (!AllowIpV4) return TIpHost(-1); - const THost& addr = Resolve(hostname, AF_INET); + const THost& addr = Resolve(hostname, AF_INET); TGuard<TMutex> lock(CacheMtx); if (addr.AddrsV4.empty()) { @@ -84,12 +84,12 @@ TIpHost TDnsCache::Get(const TString& hostname) { } NAddr::IRemoteAddrPtr TDnsCache::GetAddr( - const TString& hostname, - int family, - TIpPort port, - bool cacheOnly) { + const TString& hostname, + int family, + TIpPort port, + bool cacheOnly) { if (family != AF_INET && AllowIpV6) { - const THost& addr = Resolve(hostname, AF_INET6, cacheOnly); + const THost& addr = Resolve(hostname, AF_INET6, cacheOnly); TGuard<TMutex> lock(CacheMtx); if (!addr.AddrsV6.empty()) { @@ -104,7 +104,7 @@ NAddr::IRemoteAddrPtr TDnsCache::GetAddr( } if (family != AF_INET6 && AllowIpV4) { - const THost& addr = Resolve(hostname, AF_INET, cacheOnly); + const THost& addr = Resolve(hostname, AF_INET, cacheOnly); TGuard<TMutex> lock(CacheMtx); if (!addr.AddrsV4.empty()) { @@ -117,10 +117,10 @@ NAddr::IRemoteAddrPtr TDnsCache::GetAddr( } void TDnsCache::GetAllAddresses( - const TString& hostname, - TVector<NAddr::IRemoteAddrPtr>& addrs) { + const TString& hostname, + TVector<NAddr::IRemoteAddrPtr>& addrs) { if (AllowIpV4) { - const THost& addr4 = Resolve(hostname, AF_INET); + const THost& addr4 = Resolve(hostname, AF_INET); TGuard<TMutex> lock(CacheMtx); for (size_t i = 0; i < addr4.AddrsV4.size(); i++) { @@ -129,7 +129,7 @@ void TDnsCache::GetAllAddresses( } if (AllowIpV6) { - const THost& addr6 = Resolve(hostname, AF_INET6); + const THost& addr6 = Resolve(hostname, AF_INET6); struct sockaddr_in6 sin6; Zero(sin6); @@ -144,8 +144,8 @@ void TDnsCache::GetAllAddresses( } } -void TDnsCache::GetStats(ui64& a_cache_hits, ui64& a_cache_misses, - ui64& ptr_cache_hits, ui64& ptr_cache_misses) { +void TDnsCache::GetStats(ui64& a_cache_hits, ui64& a_cache_misses, + ui64& ptr_cache_hits, ui64& ptr_cache_misses) { TGuard<TMutex> lock(CacheMtx); a_cache_hits = ACacheHits; @@ -154,7 +154,7 @@ void TDnsCache::GetStats(ui64& a_cache_hits, ui64& a_cache_misses, ptr_cache_misses = PtrCacheMisses; } -bool TDnsCache::THost::IsStale(int family, const TDnsCache* ctx) const noexcept { +bool TDnsCache::THost::IsStale(int family, const TDnsCache* ctx) const noexcept { time_t resolved = family == AF_INET ? ResolvedV4 : ResolvedV6; time_t notfound = family == AF_INET ? NotFoundV4 : NotFoundV6; @@ -168,7 +168,7 @@ bool TDnsCache::THost::IsStale(int family, const TDnsCache* ctx) const noexcept } const TDnsCache::THost& -TDnsCache::Resolve(const TString& hostname, int family, bool cacheOnly) { +TDnsCache::Resolve(const TString& hostname, int family, bool cacheOnly) { if (!ValidateHName(hostname)) { LWPROBE(ResolveNullHost, hostname, family); return NullHost; @@ -202,7 +202,7 @@ TDnsCache::Resolve(const TString& hostname, int family, bool cacheOnly) { if (cacheOnly) return NullHost; - TAtomic& inprogress = (family == AF_INET ? p->second.InProgressV4 : p->second.InProgressV6); + TAtomic& inprogress = (family == AF_INET ? p->second.InProgressV4 : p->second.InProgressV6); { /* This way only! CacheMtx should always be taken AFTER AresMtx, @@ -212,16 +212,16 @@ TDnsCache::Resolve(const TString& hostname, int family, bool cacheOnly) { TGuard<TMutex> areslock(AresMtx); TGuard<TMutex> cachelock(CacheMtx); - if (!inprogress) { + if (!inprogress) { ares_channel chan = static_cast<ares_channel>(Channel); - TGHBNContext* ctx = new TGHBNContext(); + TGHBNContext* ctx = new TGHBNContext(); ctx->Owner = this; ctx->Hostname = hostname; ctx->Family = family; AtomicSet(inprogress, 1); ares_gethostbyname(chan, hostname.c_str(), family, - &TDnsCache::GHBNCallback, ctx); + &TDnsCache::GHBNCallback, ctx); } } @@ -231,18 +231,18 @@ TDnsCache::Resolve(const TString& hostname, int family, bool cacheOnly) { return p->second; } -bool TDnsCache::ValidateHName(const TString& name) const noexcept { +bool TDnsCache::ValidateHName(const TString& name) const noexcept { return name.size() > 0; } -const TDnsCache::TAddr& TDnsCache::ResolveAddr(const in6_addr& addr, int family) { +const TDnsCache::TAddr& TDnsCache::ResolveAddr(const in6_addr& addr, int family) { TAddrCache::iterator p; { TGuard<TMutex> lock(CacheMtx); p = AddrCache.find(addr); if (p != AddrCache.end()) { - if (TTimeKeeper::GetTime() - p->second.Resolved < EntryLifetime || TTimeKeeper::GetTime() - p->second.NotFound < NegativeLifetime) { + if (TTimeKeeper::GetTime() - p->second.Resolved < EntryLifetime || TTimeKeeper::GetTime() - p->second.NotFound < NegativeLifetime) { /* Recently resolved, just return cached value */ PtrCacheHits += 1; return p->second; @@ -263,16 +263,16 @@ const TDnsCache::TAddr& TDnsCache::ResolveAddr(const in6_addr& addr, int family) TGuard<TMutex> areslock(AresMtx); TGuard<TMutex> cachelock(CacheMtx); - if (!p->second.InProgress) { + if (!p->second.InProgress) { ares_channel chan = static_cast<ares_channel>(Channel); - TGHBAContext* ctx = new TGHBAContext(); + TGHBAContext* ctx = new TGHBAContext(); ctx->Owner = this; ctx->Addr = addr; AtomicSet(p->second.InProgress, 1); ares_gethostbyaddr(chan, &addr, - family == AF_INET ? sizeof(in_addr) : sizeof(in6_addr), - family, &TDnsCache::GHBACallback, ctx); + family == AF_INET ? sizeof(in_addr) : sizeof(in6_addr), + family, &TDnsCache::GHBACallback, ctx); } } @@ -281,7 +281,7 @@ const TDnsCache::TAddr& TDnsCache::ResolveAddr(const in6_addr& addr, int family) return p->second; } -void TDnsCache::WaitTask(TAtomic& flag) { +void TDnsCache::WaitTask(TAtomic& flag) { const TInstant start = TInstant(TTimeKeeper::GetTimeval()); while (AtomicGet(flag)) { @@ -306,11 +306,11 @@ void TDnsCache::WaitTask(TAtomic& flag) { pfd[nfds].revents = 0; if (ARES_GETSOCK_READABLE(bits, nfds)) { pfd[nfds].fd = socks[nfds]; - pfd[nfds].events |= POLLRDNORM | POLLIN; + pfd[nfds].events |= POLLRDNORM | POLLIN; } if (ARES_GETSOCK_WRITABLE(bits, nfds)) { pfd[nfds].fd = socks[nfds]; - pfd[nfds].events |= POLLWRNORM | POLLOUT; + pfd[nfds].events |= POLLWRNORM | POLLOUT; } if (pfd[nfds].events == 0) { break; @@ -343,12 +343,12 @@ void TDnsCache::WaitTask(TAtomic& flag) { } TGuard<TMutex> lock(AresMtx); ares_process_fd(chan, - pfd[i].revents & (POLLRDNORM | POLLIN) - ? pfd[i].fd - : ARES_SOCKET_BAD, - pfd[i].revents & (POLLWRNORM | POLLOUT) - ? pfd[i].fd - : ARES_SOCKET_BAD); + pfd[i].revents & (POLLRDNORM | POLLIN) + ? pfd[i].fd + : ARES_SOCKET_BAD, + pfd[i].revents & (POLLWRNORM | POLLOUT) + ? pfd[i].fd + : ARES_SOCKET_BAD); } } @@ -358,22 +358,22 @@ void TDnsCache::WaitTask(TAtomic& flag) { } } -void TDnsCache::GHBNCallback(void* arg, int status, int, struct hostent* info) { - THolder<TGHBNContext> ctx(static_cast<TGHBNContext*>(arg)); +void TDnsCache::GHBNCallback(void* arg, int status, int, struct hostent* info) { + THolder<TGHBNContext> ctx(static_cast<TGHBNContext*>(arg)); TGuard<TMutex> lock(ctx->Owner->CacheMtx); THostCache::iterator p = ctx->Owner->HostCache.find(ctx->Hostname); Y_ASSERT(p != ctx->Owner->HostCache.end()); - time_t& resolved = (ctx->Family == AF_INET ? p->second.ResolvedV4 : p->second.ResolvedV6); - time_t& notfound = (ctx->Family == AF_INET ? p->second.NotFoundV4 : p->second.NotFoundV6); - TAtomic& inprogress = (ctx->Family == AF_INET ? p->second.InProgressV4 : p->second.InProgressV6); + time_t& resolved = (ctx->Family == AF_INET ? p->second.ResolvedV4 : p->second.ResolvedV6); + time_t& notfound = (ctx->Family == AF_INET ? p->second.NotFoundV4 : p->second.NotFoundV6); + TAtomic& inprogress = (ctx->Family == AF_INET ? p->second.InProgressV4 : p->second.InProgressV6); if (status == ARES_SUCCESS) { if (info->h_addrtype == AF_INET) { p->second.AddrsV4.clear(); for (int i = 0; info->h_addr_list[i] != nullptr; i++) { - p->second.AddrsV4.push_back(*(TIpHost*)(info->h_addr_list[i])); + p->second.AddrsV4.push_back(*(TIpHost*)(info->h_addr_list[i])); } /* It is possible to ask ares for IPv6 and have IPv4 addrs instead, so take care and set V4 timers anyway. @@ -384,7 +384,7 @@ void TDnsCache::GHBNCallback(void* arg, int status, int, struct hostent* info) { } else if (info->h_addrtype == AF_INET6) { p->second.AddrsV6.clear(); for (int i = 0; info->h_addr_list[i] != nullptr; i++) { - p->second.AddrsV6.push_back(*(struct in6_addr*)(info->h_addr_list[i])); + p->second.AddrsV6.push_back(*(struct in6_addr*)(info->h_addr_list[i])); } } else { Y_FAIL("unknown address type in ares callback"); @@ -398,8 +398,8 @@ void TDnsCache::GHBNCallback(void* arg, int status, int, struct hostent* info) { AtomicSet(inprogress, 0); } -void TDnsCache::GHBACallback(void* arg, int status, int, struct hostent* info) { - THolder<TGHBAContext> ctx(static_cast<TGHBAContext*>(arg)); +void TDnsCache::GHBACallback(void* arg, int status, int, struct hostent* info) { + THolder<TGHBAContext> ctx(static_cast<TGHBAContext*>(arg)); TGuard<TMutex> lock(ctx->Owner->CacheMtx); TAddrCache::iterator p = ctx->Owner->AddrCache.find(ctx->Addr); @@ -416,17 +416,17 @@ void TDnsCache::GHBACallback(void* arg, int status, int, struct hostent* info) { AtomicSet(p->second.InProgress, 0); } -TString TDnsCache::THost::AddrsV4ToString() const { +TString TDnsCache::THost::AddrsV4ToString() const { TStringStream ss; bool first = false; for (TIpHost addr : AddrsV4) { - ss << (first ? "" : " ") << IpToString(addr); + ss << (first ? "" : " ") << IpToString(addr); first = false; } return ss.Str(); } -TString TDnsCache::THost::AddrsV6ToString() const { +TString TDnsCache::THost::AddrsV6ToString() const { TStringStream ss; bool first = false; for (in6_addr addr : AddrsV6) { @@ -436,7 +436,7 @@ TString TDnsCache::THost::AddrsV6ToString() const { sin6.sin6_addr = addr; NAddr::TIPv6Addr addr6(sin6); - ss << (first ? "" : " ") << NAddr::PrintHost(addr6); + ss << (first ? "" : " ") << NAddr::PrintHost(addr6); first = false; } return ss.Str(); diff --git a/library/cpp/actors/dnscachelib/dnscache.h b/library/cpp/actors/dnscachelib/dnscache.h index 3313a251a1..a8e80de628 100644 --- a/library/cpp/actors/dnscachelib/dnscache.h +++ b/library/cpp/actors/dnscachelib/dnscache.h @@ -24,47 +24,47 @@ public: TString GetHostByAddr(const NAddr::IRemoteAddr&); // ip in network byte order - TIpHost Get(const TString& host); + TIpHost Get(const TString& host); /* use with AF_INET, AF_INET6 or AF_UNSPEC */ - NAddr::IRemoteAddrPtr GetAddr(const TString& host, + NAddr::IRemoteAddrPtr GetAddr(const TString& host, int family, TIpPort port = 0, bool cacheOnly = false); - void GetAllAddresses(const TString& host, TVector<NAddr::IRemoteAddrPtr>&); + void GetAllAddresses(const TString& host, TVector<NAddr::IRemoteAddrPtr>&); - void GetStats(ui64& a_cache_hits, ui64& a_cache_misses, - ui64& ptr_cache_hits, ui64& ptr_cache_misses); + void GetStats(ui64& a_cache_hits, ui64& a_cache_misses, + ui64& ptr_cache_hits, ui64& ptr_cache_misses); protected: - bool ValidateHName(const TString& host) const noexcept; + bool ValidateHName(const TString& host) const noexcept; private: struct TGHBNContext { - TDnsCache* Owner; + TDnsCache* Owner; TString Hostname; int Family; }; struct TGHBAContext { - TDnsCache* Owner; + TDnsCache* Owner; in6_addr Addr; }; struct THost { - THost() noexcept { - } + THost() noexcept { + } TVector<TIpHost> AddrsV4; - time_t ResolvedV4 = 0; - time_t NotFoundV4 = 0; - TAtomic InProgressV4 = 0; + time_t ResolvedV4 = 0; + time_t NotFoundV4 = 0; + TAtomic InProgressV4 = 0; TVector<in6_addr> AddrsV6; - time_t ResolvedV6 = 0; - time_t NotFoundV6 = 0; - TAtomic InProgressV6 = 0; + time_t ResolvedV6 = 0; + time_t NotFoundV6 = 0; + TAtomic InProgressV6 = 0; TString AddrsV4ToString() const; TString AddrsV6ToString() const; @@ -75,16 +75,16 @@ private: typedef TMap<TString, THost> THostCache; struct TAddr { - TString Hostname; - time_t Resolved = 0; - time_t NotFound = 0; - TAtomic InProgress = 0; + TString Hostname; + time_t Resolved = 0; + time_t NotFound = 0; + TAtomic InProgress = 0; }; /* IRemoteAddr is annoingly hard to use, so I'll use in6_addr as key * and put v4 addrs in it. */ struct TAddrCmp { - bool operator()(const in6_addr& left, const in6_addr& right) const { + bool operator()(const in6_addr& left, const in6_addr& right) const { for (size_t i = 0; i < sizeof(left); i++) { if (left.s6_addr[i] < right.s6_addr[i]) { return true; @@ -98,17 +98,17 @@ private: }; typedef TMap<in6_addr, TAddr, TAddrCmp> TAddrCache; - const THost& Resolve(const TString&, int family, bool cacheOnly = false); + const THost& Resolve(const TString&, int family, bool cacheOnly = false); - const TAddr& ResolveAddr(const in6_addr&, int family); + const TAddr& ResolveAddr(const in6_addr&, int family); - void WaitTask(TAtomic&); + void WaitTask(TAtomic&); - static void GHBNCallback(void* arg, int status, int timeouts, - struct hostent* info); + static void GHBNCallback(void* arg, int status, int timeouts, + struct hostent* info); - static void GHBACallback(void* arg, int status, int timeouts, - struct hostent* info); + static void GHBACallback(void* arg, int status, int timeouts, + struct hostent* info); const time_t EntryLifetime; const time_t NegativeLifetime; @@ -127,7 +127,7 @@ private: const static THost NullHost; TMutex AresMtx; - void* Channel; + void* Channel; struct TAresLibInit { TAresLibInit() { diff --git a/library/cpp/actors/dnscachelib/probes.h b/library/cpp/actors/dnscachelib/probes.h index 313b7b8712..9e4781817f 100644 --- a/library/cpp/actors/dnscachelib/probes.h +++ b/library/cpp/actors/dnscachelib/probes.h @@ -2,34 +2,34 @@ #include <library/cpp/lwtrace/all.h> -#define DNSCACHELIB_PROVIDER(PROBE, EVENT, GROUPS, TYPES, NAMES) \ - PROBE(Created, GROUPS(), TYPES(), NAMES()) \ - PROBE(Destroyed, GROUPS(), TYPES(), NAMES()) \ - PROBE(AresInitFailed, GROUPS(), TYPES(), NAMES()) \ - PROBE(FamilyMismatch, \ - GROUPS(), \ - TYPES(int, bool, bool), \ - NAMES("family", "allowIpV4", "allowIpV6")) \ - PROBE(ResolveNullHost, \ - GROUPS(), \ - TYPES(TString, int), \ - NAMES("hostname", "family")) \ - PROBE(ResolveFromCache, \ - GROUPS(), \ - TYPES(TString, int, TString, TString, ui64), \ - NAMES("hostname", "family", "addrsV4", "addrsV6", "aCacheHits")) \ - PROBE(ResolveDone, \ - GROUPS(), \ - TYPES(TString, int, TString, TString), \ - NAMES("hostname", "family", "addrsV4", "addrsV6")) \ - PROBE(ResolveCacheTimeout, \ - GROUPS(), \ - TYPES(TString), \ - NAMES("hostname")) \ - PROBE(ResolveCacheNew, \ - GROUPS(), \ - TYPES(TString), \ - NAMES("hostname")) \ +#define DNSCACHELIB_PROVIDER(PROBE, EVENT, GROUPS, TYPES, NAMES) \ + PROBE(Created, GROUPS(), TYPES(), NAMES()) \ + PROBE(Destroyed, GROUPS(), TYPES(), NAMES()) \ + PROBE(AresInitFailed, GROUPS(), TYPES(), NAMES()) \ + PROBE(FamilyMismatch, \ + GROUPS(), \ + TYPES(int, bool, bool), \ + NAMES("family", "allowIpV4", "allowIpV6")) \ + PROBE(ResolveNullHost, \ + GROUPS(), \ + TYPES(TString, int), \ + NAMES("hostname", "family")) \ + PROBE(ResolveFromCache, \ + GROUPS(), \ + TYPES(TString, int, TString, TString, ui64), \ + NAMES("hostname", "family", "addrsV4", "addrsV6", "aCacheHits")) \ + PROBE(ResolveDone, \ + GROUPS(), \ + TYPES(TString, int, TString, TString), \ + NAMES("hostname", "family", "addrsV4", "addrsV6")) \ + PROBE(ResolveCacheTimeout, \ + GROUPS(), \ + TYPES(TString), \ + NAMES("hostname")) \ + PROBE(ResolveCacheNew, \ + GROUPS(), \ + TYPES(TString), \ + NAMES("hostname")) \ /**/ LWTRACE_DECLARE_PROVIDER(DNSCACHELIB_PROVIDER) diff --git a/library/cpp/actors/dnscachelib/timekeeper.h b/library/cpp/actors/dnscachelib/timekeeper.h index 0528d8549c..daf62ebeaa 100644 --- a/library/cpp/actors/dnscachelib/timekeeper.h +++ b/library/cpp/actors/dnscachelib/timekeeper.h @@ -21,17 +21,17 @@ public: return Singleton<TTimeKeeper>()->CurrentTime.tv_sec; } - static const struct timeval& GetTimeval(void) { + static const struct timeval& GetTimeval(void) { return Singleton<TTimeKeeper>()->CurrentTime; } TTimeKeeper() - : Thread(&TTimeKeeper::Worker, this) + : Thread(&TTimeKeeper::Worker, this) { ConstTime = !!GetEnv("TEST_TIME"); if (ConstTime) { try { - CurrentTime.tv_sec = FromString<ui32>(GetEnv("TEST_TIME")); + CurrentTime.tv_sec = FromString<ui32>(GetEnv("TEST_TIME")); } catch (TFromStringException exc) { ConstTime = false; } @@ -56,8 +56,8 @@ private: TSystemEvent Exit; TThread Thread; - static void* Worker(void* arg) { - TTimeKeeper* owner = static_cast<TTimeKeeper*>(arg); + static void* Worker(void* arg) { + TTimeKeeper* owner = static_cast<TTimeKeeper*>(arg); do { /* Race condition may occur here but locking looks too expensive */ diff --git a/library/cpp/actors/helpers/mon_histogram_helper.h b/library/cpp/actors/helpers/mon_histogram_helper.h index a9a57e3823..1e28ba31de 100644 --- a/library/cpp/actors/helpers/mon_histogram_helper.h +++ b/library/cpp/actors/helpers/mon_histogram_helper.h @@ -5,48 +5,48 @@ #include <util/string/cast.h> namespace NActors { - namespace NMon { - class THistogramCounterHelper { - public: - THistogramCounterHelper() - : FirstBucketVal(0) - , BucketCount(0) - { - } + namespace NMon { + class THistogramCounterHelper { + public: + THistogramCounterHelper() + : FirstBucketVal(0) + , BucketCount(0) + { + } THistogramCounterHelper(const THistogramCounterHelper&) = default; void Init(NMonitoring::TDynamicCounters* group, const TString& baseName, const TString& unit, ui64 firstBucket, ui64 bucketCnt, bool useSensorLabelName = true) { - Y_ASSERT(FirstBucketVal == 0); - Y_ASSERT(BucketCount == 0); + Y_ASSERT(FirstBucketVal == 0); + Y_ASSERT(BucketCount == 0); - FirstBucketVal = firstBucket; - BucketCount = bucketCnt; - BucketsHolder.reserve(BucketCount); - Buckets.reserve(BucketCount); - for (size_t i = 0; i < BucketCount; ++i) { - TString bucketName = GetBucketName(i) + " " + unit; + FirstBucketVal = firstBucket; + BucketCount = bucketCnt; + BucketsHolder.reserve(BucketCount); + Buckets.reserve(BucketCount); + for (size_t i = 0; i < BucketCount; ++i) { + TString bucketName = GetBucketName(i) + " " + unit; auto labelName = useSensorLabelName ? "sensor" : "name"; BucketsHolder.push_back(group->GetSubgroup(labelName, baseName)->GetNamedCounter("range", bucketName, true)); - Buckets.push_back(BucketsHolder.back().Get()); - } - } + Buckets.push_back(BucketsHolder.back().Get()); + } + } - void Add(ui64 val) { - Y_ASSERT(FirstBucketVal != 0); - Y_ASSERT(BucketCount != 0); - Y_VERIFY(val <= (1ULL << 63ULL)); - size_t ind = 0; - if (val > FirstBucketVal) { - ind = GetValueBitCount((2 * val - 1) / FirstBucketVal) - 1; - if (ind >= BucketCount) { - ind = BucketCount - 1; - } - } - Buckets[ind]->Inc(); - } + void Add(ui64 val) { + Y_ASSERT(FirstBucketVal != 0); + Y_ASSERT(BucketCount != 0); + Y_VERIFY(val <= (1ULL << 63ULL)); + size_t ind = 0; + if (val > FirstBucketVal) { + ind = GetValueBitCount((2 * val - 1) / FirstBucketVal) - 1; + if (ind >= BucketCount) { + ind = BucketCount - 1; + } + } + Buckets[ind]->Inc(); + } ui64 GetBucketCount() const { return BucketCount; @@ -62,25 +62,25 @@ namespace NActors { *Buckets[index] = value; } - private: - TString GetBucketName(size_t ind) const { - Y_ASSERT(FirstBucketVal != 0); - Y_ASSERT(BucketCount != 0); - Y_ASSERT(ind < BucketCount); - if (ind + 1 < BucketCount) { - return ToString<ui64>(FirstBucketVal << ind); - } else { - // Last slot is up to +INF - return "INF"; - } + private: + TString GetBucketName(size_t ind) const { + Y_ASSERT(FirstBucketVal != 0); + Y_ASSERT(BucketCount != 0); + Y_ASSERT(ind < BucketCount); + if (ind + 1 < BucketCount) { + return ToString<ui64>(FirstBucketVal << ind); + } else { + // Last slot is up to +INF + return "INF"; + } } - private: - ui64 FirstBucketVal; - ui64 BucketCount; - TVector<NMonitoring::TDynamicCounters::TCounterPtr> BucketsHolder; + private: + ui64 FirstBucketVal; + ui64 BucketCount; + TVector<NMonitoring::TDynamicCounters::TCounterPtr> BucketsHolder; TVector<NMonitoring::TDeprecatedCounter*> Buckets; - }; - + }; + } -} +} diff --git a/library/cpp/actors/interconnect/event_holder_pool.h b/library/cpp/actors/interconnect/event_holder_pool.h index b6090a3bc8..9fe5e03be6 100644 --- a/library/cpp/actors/interconnect/event_holder_pool.h +++ b/library/cpp/actors/interconnect/event_holder_pool.h @@ -117,7 +117,7 @@ namespace NActors { } private: - TEvFreeItems* GetPendingEvent() { + TEvFreeItems* GetPendingEvent() { if (!PendingFreeEvent) { PendingFreeEvent.Reset(new TEvFreeItems); } @@ -125,4 +125,4 @@ namespace NActors { } }; -} +} diff --git a/library/cpp/actors/interconnect/events_local.h b/library/cpp/actors/interconnect/events_local.h index 8a46ffd535..7880ca283f 100644 --- a/library/cpp/actors/interconnect/events_local.h +++ b/library/cpp/actors/interconnect/events_local.h @@ -11,324 +11,324 @@ #include "types.h" namespace NActors { - struct TProgramInfo { - ui64 PID = 0; - ui64 StartTime = 0; - ui64 Serial = 0; - }; - - enum class ENetwork : ui32 { - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // local messages - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - Start = EventSpaceBegin(TEvents::ES_INTERCONNECT_TCP), - - SocketReadyRead = Start, - SocketReadyWrite, - SocketError, - Connect, - Disconnect, - IncomingConnection, - HandshakeAsk, - HandshakeAck, + struct TProgramInfo { + ui64 PID = 0; + ui64 StartTime = 0; + ui64 Serial = 0; + }; + + enum class ENetwork : ui32 { + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // local messages + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + Start = EventSpaceBegin(TEvents::ES_INTERCONNECT_TCP), + + SocketReadyRead = Start, + SocketReadyWrite, + SocketError, + Connect, + Disconnect, + IncomingConnection, + HandshakeAsk, + HandshakeAck, HandshakeNak, - HandshakeDone, - HandshakeFail, - Kick, - Flush, - NodeInfo, - BunchOfEventsToDestroy, - HandshakeRequest, - HandshakeReplyOK, - HandshakeReplyError, - ResolveAddress, - AddressInfo, - ResolveError, - HTTPStreamStatus, - HTTPSendContent, - ConnectProtocolWakeup, - HTTPProtocolRetry, - EvPollerRegister, + HandshakeDone, + HandshakeFail, + Kick, + Flush, + NodeInfo, + BunchOfEventsToDestroy, + HandshakeRequest, + HandshakeReplyOK, + HandshakeReplyError, + ResolveAddress, + AddressInfo, + ResolveError, + HTTPStreamStatus, + HTTPSendContent, + ConnectProtocolWakeup, + HTTPProtocolRetry, + EvPollerRegister, EvPollerRegisterResult, EvPollerReady, - EvUpdateFromInputSession, + EvUpdateFromInputSession, EvConfirmUpdate, - EvSessionBufferSizeRequest, - EvSessionBufferSizeResponse, + EvSessionBufferSizeRequest, + EvSessionBufferSizeResponse, EvProcessPingRequest, EvGetSecureSocket, EvSecureSocket, - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // nonlocal messages; their indices must be preserved in order to work properly while doing rolling update - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - - // interconnect load test message - EvLoadMessage = Start + 256, - }; - - struct TEvSocketReadyRead: public TEventLocal<TEvSocketReadyRead, ui32(ENetwork::SocketReadyRead)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvSocketReadyRead, "Network: TEvSocketReadyRead") - }; - - struct TEvSocketReadyWrite: public TEventLocal<TEvSocketReadyWrite, ui32(ENetwork::SocketReadyWrite)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvSocketReadyWrite, "Network: TEvSocketReadyWrite") - }; - - struct TEvSocketError: public TEventLocal<TEvSocketError, ui32(ENetwork::SocketError)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvSocketError, ::strerror(Error)) - TString GetReason() const { - return ::strerror(Error); - } - const int Error; - TIntrusivePtr<NInterconnect::TStreamSocket> Socket; - - TEvSocketError(int error, TIntrusivePtr<NInterconnect::TStreamSocket> sock) - : Error(error) - , Socket(std::move(sock)) - { - } - }; - - struct TEvSocketConnect: public TEventLocal<TEvSocketConnect, ui32(ENetwork::Connect)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvSocketConnect, "Network: TEvSocketConnect") - }; - - struct TEvSocketDisconnect: public TEventLocal<TEvSocketDisconnect, ui32(ENetwork::Disconnect)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvSocketDisconnect, "Network: TEvSocketDisconnect") + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // nonlocal messages; their indices must be preserved in order to work properly while doing rolling update + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + // interconnect load test message + EvLoadMessage = Start + 256, + }; + + struct TEvSocketReadyRead: public TEventLocal<TEvSocketReadyRead, ui32(ENetwork::SocketReadyRead)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvSocketReadyRead, "Network: TEvSocketReadyRead") + }; + + struct TEvSocketReadyWrite: public TEventLocal<TEvSocketReadyWrite, ui32(ENetwork::SocketReadyWrite)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvSocketReadyWrite, "Network: TEvSocketReadyWrite") + }; + + struct TEvSocketError: public TEventLocal<TEvSocketError, ui32(ENetwork::SocketError)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvSocketError, ::strerror(Error)) + TString GetReason() const { + return ::strerror(Error); + } + const int Error; + TIntrusivePtr<NInterconnect::TStreamSocket> Socket; + + TEvSocketError(int error, TIntrusivePtr<NInterconnect::TStreamSocket> sock) + : Error(error) + , Socket(std::move(sock)) + { + } + }; + + struct TEvSocketConnect: public TEventLocal<TEvSocketConnect, ui32(ENetwork::Connect)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvSocketConnect, "Network: TEvSocketConnect") + }; + + struct TEvSocketDisconnect: public TEventLocal<TEvSocketDisconnect, ui32(ENetwork::Disconnect)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvSocketDisconnect, "Network: TEvSocketDisconnect") TDisconnectReason Reason; TEvSocketDisconnect(TDisconnectReason reason) : Reason(std::move(reason)) - { - } - }; + { + } + }; - struct TEvHandshakeAsk: public TEventLocal<TEvHandshakeAsk, ui32(ENetwork::HandshakeAsk)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeAsk, "Network: TEvHandshakeAsk") + struct TEvHandshakeAsk: public TEventLocal<TEvHandshakeAsk, ui32(ENetwork::HandshakeAsk)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeAsk, "Network: TEvHandshakeAsk") TEvHandshakeAsk(const TActorId& self, const TActorId& peer, - ui64 counter) - : Self(self) - , Peer(peer) - , Counter(counter) - { - } + ui64 counter) + : Self(self) + , Peer(peer) + , Counter(counter) + { + } const TActorId Self; const TActorId Peer; - const ui64 Counter; - }; + const ui64 Counter; + }; - struct TEvHandshakeAck: public TEventLocal<TEvHandshakeAck, ui32(ENetwork::HandshakeAck)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeAck, "Network: TEvHandshakeAck") + struct TEvHandshakeAck: public TEventLocal<TEvHandshakeAck, ui32(ENetwork::HandshakeAck)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeAck, "Network: TEvHandshakeAck") TEvHandshakeAck(const TActorId& self, ui64 nextPacket, TSessionParams params) - : Self(self) - , NextPacket(nextPacket) + : Self(self) + , NextPacket(nextPacket) , Params(std::move(params)) {} const TActorId Self; - const ui64 NextPacket; + const ui64 NextPacket; const TSessionParams Params; - }; + }; struct TEvHandshakeNak : TEventLocal<TEvHandshakeNak, ui32(ENetwork::HandshakeNak)> { DEFINE_SIMPLE_LOCAL_EVENT(TEvSocketReadyRead, "Network: TEvHandshakeNak") }; - struct TEvHandshakeRequest - : public TEventLocal<TEvHandshakeRequest, - ui32(ENetwork::HandshakeRequest)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeRequest, - "Network: TEvHandshakeRequest") + struct TEvHandshakeRequest + : public TEventLocal<TEvHandshakeRequest, + ui32(ENetwork::HandshakeRequest)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeRequest, + "Network: TEvHandshakeRequest") - NActorsInterconnect::THandshakeRequest Record; - }; + NActorsInterconnect::THandshakeRequest Record; + }; - struct TEvHandshakeReplyOK - : public TEventLocal<TEvHandshakeReplyOK, - ui32(ENetwork::HandshakeReplyOK)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeReplyOK, - "Network: TEvHandshakeReplyOK") + struct TEvHandshakeReplyOK + : public TEventLocal<TEvHandshakeReplyOK, + ui32(ENetwork::HandshakeReplyOK)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeReplyOK, + "Network: TEvHandshakeReplyOK") - NActorsInterconnect::THandshakeReply Record; - }; + NActorsInterconnect::THandshakeReply Record; + }; - struct TEvHandshakeReplyError - : public TEventLocal<TEvHandshakeReplyError, - ui32(ENetwork::HandshakeReplyError)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeReplyError, - "Network: TEvHandshakeReplyError") + struct TEvHandshakeReplyError + : public TEventLocal<TEvHandshakeReplyError, + ui32(ENetwork::HandshakeReplyError)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeReplyError, + "Network: TEvHandshakeReplyError") - TEvHandshakeReplyError(TString error) { - Record.SetErrorExplaination(error); - } + TEvHandshakeReplyError(TString error) { + Record.SetErrorExplaination(error); + } - NActorsInterconnect::THandshakeReply Record; - }; + NActorsInterconnect::THandshakeReply Record; + }; - struct TEvIncomingConnection: public TEventLocal<TEvIncomingConnection, ui32(ENetwork::IncomingConnection)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvIncomingConnection, "Network: TEvIncomingConnection") - TIntrusivePtr<NInterconnect::TStreamSocket> Socket; - NInterconnect::TAddress Address; + struct TEvIncomingConnection: public TEventLocal<TEvIncomingConnection, ui32(ENetwork::IncomingConnection)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvIncomingConnection, "Network: TEvIncomingConnection") + TIntrusivePtr<NInterconnect::TStreamSocket> Socket; + NInterconnect::TAddress Address; TEvIncomingConnection(TIntrusivePtr<NInterconnect::TStreamSocket> socket, NInterconnect::TAddress address) : Socket(std::move(socket)) , Address(std::move(address)) {} - }; + }; - struct TEvHandshakeDone: public TEventLocal<TEvHandshakeDone, ui32(ENetwork::HandshakeDone)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeDone, "Network: TEvHandshakeDone") + struct TEvHandshakeDone: public TEventLocal<TEvHandshakeDone, ui32(ENetwork::HandshakeDone)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeDone, "Network: TEvHandshakeDone") - TEvHandshakeDone( + TEvHandshakeDone( TIntrusivePtr<NInterconnect::TStreamSocket> socket, const TActorId& peer, const TActorId& self, ui64 nextPacket, TAutoPtr<TProgramInfo>&& programInfo, TSessionParams params) - : Socket(std::move(socket)) - , Peer(peer) - , Self(self) - , NextPacket(nextPacket) - , ProgramInfo(std::move(programInfo)) + : Socket(std::move(socket)) + , Peer(peer) + , Self(self) + , NextPacket(nextPacket) + , ProgramInfo(std::move(programInfo)) , Params(std::move(params)) - { - } + { + } - TIntrusivePtr<NInterconnect::TStreamSocket> Socket; + TIntrusivePtr<NInterconnect::TStreamSocket> Socket; const TActorId Peer; const TActorId Self; - const ui64 NextPacket; - TAutoPtr<TProgramInfo> ProgramInfo; + const ui64 NextPacket; + TAutoPtr<TProgramInfo> ProgramInfo; const TSessionParams Params; - }; - - struct TEvHandshakeFail: public TEventLocal<TEvHandshakeFail, ui32(ENetwork::HandshakeFail)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeFail, "Network: TEvHandshakeFail") + }; - enum EnumHandshakeFail { - HANDSHAKE_FAIL_TRANSIENT, - HANDSHAKE_FAIL_PERMANENT, - HANDSHAKE_FAIL_SESSION_MISMATCH, - }; + struct TEvHandshakeFail: public TEventLocal<TEvHandshakeFail, ui32(ENetwork::HandshakeFail)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvHandshakeFail, "Network: TEvHandshakeFail") + enum EnumHandshakeFail { + HANDSHAKE_FAIL_TRANSIENT, + HANDSHAKE_FAIL_PERMANENT, + HANDSHAKE_FAIL_SESSION_MISMATCH, + }; + TEvHandshakeFail(EnumHandshakeFail temporary, TString explanation) - : Temporary(temporary) - , Explanation(std::move(explanation)) - { - } - - const EnumHandshakeFail Temporary; - const TString Explanation; - }; - - struct TEvKick: public TEventLocal<TEvKick, ui32(ENetwork::Kick)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvKick, "Network: TEvKick") - }; - - struct TEvFlush: public TEventLocal<TEvFlush, ui32(ENetwork::Flush)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvFlush, "Network: TEvFlush") - }; - - struct TEvLocalNodeInfo - : public TEventLocal<TEvLocalNodeInfo, ui32(ENetwork::NodeInfo)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvLocalNodeInfo, "Network: TEvLocalNodeInfo") - - ui32 NodeId; - NAddr::IRemoteAddrPtr Address; - }; - - struct TEvBunchOfEventsToDestroy : TEventLocal<TEvBunchOfEventsToDestroy, ui32(ENetwork::BunchOfEventsToDestroy)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvBunchOfEventsToDestroy, - "Network: TEvBunchOfEventsToDestroy") - - TEvBunchOfEventsToDestroy(TDeque<TAutoPtr<IEventBase>> events) - : Events(std::move(events)) - { - } - - TDeque<TAutoPtr<IEventBase>> Events; - }; - - struct TEvResolveAddress - : public TEventLocal<TEvResolveAddress, ui32(ENetwork::ResolveAddress)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvResolveAddress, "Network: TEvResolveAddress") - - TString Address; - ui16 Port; - }; - - struct TEvAddressInfo - : public TEventLocal<TEvAddressInfo, ui32(ENetwork::AddressInfo)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvAddressInfo, "Network: TEvAddressInfo") - - NAddr::IRemoteAddrPtr Address; - }; - - struct TEvResolveError - : public TEventLocal<TEvResolveError, ui32(ENetwork::ResolveError)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvResolveError, "Network: TEvResolveError") - - TString Explain; - }; - - struct TEvHTTPStreamStatus - : public TEventLocal<TEvHTTPStreamStatus, ui32(ENetwork::HTTPStreamStatus)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvHTTPStreamStatus, - "Network: TEvHTTPStreamStatus") - enum EStatus { - READY, - COMPLETE, - ERROR, - }; - - EStatus Status; - TString Error; - TString HttpHeaders; - }; - - struct TEvHTTPSendContent - : public TEventLocal<TEvHTTPSendContent, ui32(ENetwork::HTTPSendContent)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvHTTPSendContent, "Network: TEvHTTPSendContent") - - const char* Data; - size_t Len; - bool Last; - }; - - struct TEvConnectWakeup - : public TEventLocal<TEvConnectWakeup, - ui32(ENetwork::ConnectProtocolWakeup)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvConnectWakeup, "Protocols: TEvConnectWakeup") - }; - - struct TEvHTTPProtocolRetry - : public TEventLocal<TEvHTTPProtocolRetry, - ui32(ENetwork::HTTPProtocolRetry)> { - DEFINE_SIMPLE_LOCAL_EVENT(TEvHTTPProtocolRetry, - "Protocols: TEvHTTPProtocolRetry") - }; - - struct TEvLoadMessage - : TEventPB<TEvLoadMessage, NActorsInterconnect::TEvLoadMessage, static_cast<ui32>(ENetwork::EvLoadMessage)> { - TEvLoadMessage() = default; - - template <typename TContainer> + : Temporary(temporary) + , Explanation(std::move(explanation)) + { + } + + const EnumHandshakeFail Temporary; + const TString Explanation; + }; + + struct TEvKick: public TEventLocal<TEvKick, ui32(ENetwork::Kick)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvKick, "Network: TEvKick") + }; + + struct TEvFlush: public TEventLocal<TEvFlush, ui32(ENetwork::Flush)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvFlush, "Network: TEvFlush") + }; + + struct TEvLocalNodeInfo + : public TEventLocal<TEvLocalNodeInfo, ui32(ENetwork::NodeInfo)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvLocalNodeInfo, "Network: TEvLocalNodeInfo") + + ui32 NodeId; + NAddr::IRemoteAddrPtr Address; + }; + + struct TEvBunchOfEventsToDestroy : TEventLocal<TEvBunchOfEventsToDestroy, ui32(ENetwork::BunchOfEventsToDestroy)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvBunchOfEventsToDestroy, + "Network: TEvBunchOfEventsToDestroy") + + TEvBunchOfEventsToDestroy(TDeque<TAutoPtr<IEventBase>> events) + : Events(std::move(events)) + { + } + + TDeque<TAutoPtr<IEventBase>> Events; + }; + + struct TEvResolveAddress + : public TEventLocal<TEvResolveAddress, ui32(ENetwork::ResolveAddress)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvResolveAddress, "Network: TEvResolveAddress") + + TString Address; + ui16 Port; + }; + + struct TEvAddressInfo + : public TEventLocal<TEvAddressInfo, ui32(ENetwork::AddressInfo)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvAddressInfo, "Network: TEvAddressInfo") + + NAddr::IRemoteAddrPtr Address; + }; + + struct TEvResolveError + : public TEventLocal<TEvResolveError, ui32(ENetwork::ResolveError)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvResolveError, "Network: TEvResolveError") + + TString Explain; + }; + + struct TEvHTTPStreamStatus + : public TEventLocal<TEvHTTPStreamStatus, ui32(ENetwork::HTTPStreamStatus)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvHTTPStreamStatus, + "Network: TEvHTTPStreamStatus") + enum EStatus { + READY, + COMPLETE, + ERROR, + }; + + EStatus Status; + TString Error; + TString HttpHeaders; + }; + + struct TEvHTTPSendContent + : public TEventLocal<TEvHTTPSendContent, ui32(ENetwork::HTTPSendContent)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvHTTPSendContent, "Network: TEvHTTPSendContent") + + const char* Data; + size_t Len; + bool Last; + }; + + struct TEvConnectWakeup + : public TEventLocal<TEvConnectWakeup, + ui32(ENetwork::ConnectProtocolWakeup)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvConnectWakeup, "Protocols: TEvConnectWakeup") + }; + + struct TEvHTTPProtocolRetry + : public TEventLocal<TEvHTTPProtocolRetry, + ui32(ENetwork::HTTPProtocolRetry)> { + DEFINE_SIMPLE_LOCAL_EVENT(TEvHTTPProtocolRetry, + "Protocols: TEvHTTPProtocolRetry") + }; + + struct TEvLoadMessage + : TEventPB<TEvLoadMessage, NActorsInterconnect::TEvLoadMessage, static_cast<ui32>(ENetwork::EvLoadMessage)> { + TEvLoadMessage() = default; + + template <typename TContainer> TEvLoadMessage(const TContainer& route, const TString& id, const TString* payload) { for (const TActorId& actorId : route) { - auto* hop = Record.AddHops(); + auto* hop = Record.AddHops(); if (actorId) { ActorIdToProto(actorId, hop->MutableNextHop()); } - } - Record.SetId(id); - if (payload) { + } + Record.SetId(id); + if (payload) { Record.SetPayload(*payload); - } + } } template <typename TContainer> @@ -342,39 +342,39 @@ namespace NActors { Record.SetId(id); AddPayload(std::move(payload)); } - }; + }; - struct TEvUpdateFromInputSession : TEventLocal<TEvUpdateFromInputSession, static_cast<ui32>(ENetwork::EvUpdateFromInputSession)> { - ui64 ConfirmedByInput; // latest Confirm value from processed input packet - ui64 NumDataBytes; + struct TEvUpdateFromInputSession : TEventLocal<TEvUpdateFromInputSession, static_cast<ui32>(ENetwork::EvUpdateFromInputSession)> { + ui64 ConfirmedByInput; // latest Confirm value from processed input packet + ui64 NumDataBytes; TDuration Ping; TEvUpdateFromInputSession(ui64 confirmedByInput, ui64 numDataBytes, TDuration ping) - : ConfirmedByInput(confirmedByInput) - , NumDataBytes(numDataBytes) + : ConfirmedByInput(confirmedByInput) + , NumDataBytes(numDataBytes) , Ping(ping) - { - } - }; + { + } + }; struct TEvConfirmUpdate : TEventLocal<TEvConfirmUpdate, static_cast<ui32>(ENetwork::EvConfirmUpdate)> {}; - struct TEvSessionBufferSizeRequest : TEventLocal<TEvSessionBufferSizeRequest, static_cast<ui32>(ENetwork::EvSessionBufferSizeRequest)> { - //DEFINE_SIMPLE_LOCAL_EVENT(TEvSessionBufferSizeRequest, "Session: TEvSessionBufferSizeRequest") - DEFINE_SIMPLE_LOCAL_EVENT(TEvSessionBufferSizeRequest, "Network: TEvSessionBufferSizeRequest"); - }; + struct TEvSessionBufferSizeRequest : TEventLocal<TEvSessionBufferSizeRequest, static_cast<ui32>(ENetwork::EvSessionBufferSizeRequest)> { + //DEFINE_SIMPLE_LOCAL_EVENT(TEvSessionBufferSizeRequest, "Session: TEvSessionBufferSizeRequest") + DEFINE_SIMPLE_LOCAL_EVENT(TEvSessionBufferSizeRequest, "Network: TEvSessionBufferSizeRequest"); + }; - struct TEvSessionBufferSizeResponse : TEventLocal<TEvSessionBufferSizeResponse, static_cast<ui32>(ENetwork::EvSessionBufferSizeResponse)> { + struct TEvSessionBufferSizeResponse : TEventLocal<TEvSessionBufferSizeResponse, static_cast<ui32>(ENetwork::EvSessionBufferSizeResponse)> { TEvSessionBufferSizeResponse(const TActorId& sessionId, ui64 outputBufferSize) - : SessionID(sessionId) - , BufferSize(outputBufferSize) - { - } + : SessionID(sessionId) + , BufferSize(outputBufferSize) + { + } TActorId SessionID; - ui64 BufferSize; - }; + ui64 BufferSize; + }; struct TEvProcessPingRequest : TEventLocal<TEvProcessPingRequest, static_cast<ui32>(ENetwork::EvProcessPingRequest)> { const ui64 Payload; @@ -400,4 +400,4 @@ namespace NActors { {} }; -} +} diff --git a/library/cpp/actors/interconnect/interconnect.h b/library/cpp/actors/interconnect/interconnect.h index 225a5243fd..79f1fd1184 100644 --- a/library/cpp/actors/interconnect/interconnect.h +++ b/library/cpp/actors/interconnect/interconnect.h @@ -6,122 +6,122 @@ #include <util/network/address.h> namespace NActors { - struct TInterconnectGlobalState: public TThrRefBase { - TString SelfAddress; - ui32 SelfPort; + struct TInterconnectGlobalState: public TThrRefBase { + TString SelfAddress; + ui32 SelfPort; TVector<TActorId> GlobalNameservers; // todo: add some info about (like expected reply time) - }; + }; - struct TInterconnectProxySetup: public TThrRefBase { - // synchronous (session -> proxy) - struct IProxy : TNonCopyable { - virtual ~IProxy() { - } + struct TInterconnectProxySetup: public TThrRefBase { + // synchronous (session -> proxy) + struct IProxy : TNonCopyable { + virtual ~IProxy() { + } - virtual void ActivateSession(const TActorContext& ctx) = 0; // session activated - virtual void DetachSession(const TActorContext& ctx) = 0; // session is dead - }; + virtual void ActivateSession(const TActorContext& ctx) = 0; // session activated + virtual void DetachSession(const TActorContext& ctx) = 0; // session is dead + }; - // synchronous (proxy -> session) - struct ISession : TNonCopyable { - virtual ~ISession() { - } + // synchronous (proxy -> session) + struct ISession : TNonCopyable { + virtual ~ISession() { + } - virtual void DetachSession(const TActorContext& ownerCtx, const TActorContext& sessionCtx) = 0; // kill yourself - virtual void ForwardPacket(TAutoPtr<IEventHandle>& ev, const TActorContext& ownerCtx, const TActorContext& sessionCtx) = 0; // receive packet for forward - virtual void Connect(const TActorContext& ownerCtx, const TActorContext& sessionCtx) = 0; // begin connection - virtual bool ReceiveIncomingSession(TAutoPtr<IEventHandle>& ev, const TActorContext& ownerCtx, const TActorContext& sessionCtx) = 0; // handle incoming session, if returns true - then session is dead and must be recreated with new one - }; + virtual void DetachSession(const TActorContext& ownerCtx, const TActorContext& sessionCtx) = 0; // kill yourself + virtual void ForwardPacket(TAutoPtr<IEventHandle>& ev, const TActorContext& ownerCtx, const TActorContext& sessionCtx) = 0; // receive packet for forward + virtual void Connect(const TActorContext& ownerCtx, const TActorContext& sessionCtx) = 0; // begin connection + virtual bool ReceiveIncomingSession(TAutoPtr<IEventHandle>& ev, const TActorContext& ownerCtx, const TActorContext& sessionCtx) = 0; // handle incoming session, if returns true - then session is dead and must be recreated with new one + }; - ui32 DestinationNode; + ui32 DestinationNode; - TString StaticAddress; // if set - would be used as main destination address - int StaticPort; + TString StaticAddress; // if set - would be used as main destination address + int StaticPort; - TIntrusivePtr<TInterconnectGlobalState> GlobalState; + TIntrusivePtr<TInterconnectGlobalState> GlobalState; virtual IActor* CreateSession(const TActorId& ownerId, IProxy* owner) = 0; // returned actor is session and would be attached to same mailbox as proxy to allow sync calls - virtual TActorSetupCmd CreateAcceptor() = 0; - }; + virtual TActorSetupCmd CreateAcceptor() = 0; + }; - struct TNameserverSetup { + struct TNameserverSetup { TActorId ServiceID; - TIntrusivePtr<TInterconnectGlobalState> GlobalState; - }; + TIntrusivePtr<TInterconnectGlobalState> GlobalState; + }; - struct TTableNameserverSetup: public TThrRefBase { - struct TNodeInfo { - TString Address; - TString Host; - TString ResolveHost; - ui16 Port; + struct TTableNameserverSetup: public TThrRefBase { + struct TNodeInfo { + TString Address; + TString Host; + TString ResolveHost; + ui16 Port; TNodeLocation Location; - TString& first; - ui16& second; + TString& first; + ui16& second; - TNodeInfo() - : first(Address) - , second(Port) - { - } + TNodeInfo() + : first(Address) + , second(Port) + { + } TNodeInfo(const TNodeInfo&) = default; - // for testing purposes only - TNodeInfo(const TString& address, const TString& host, ui16 port) - : TNodeInfo() - { - Address = address; - Host = host; - ResolveHost = host; - Port = port; - } - - TNodeInfo(const TString& address, - const TString& host, - const TString& resolveHost, - ui16 port, + // for testing purposes only + TNodeInfo(const TString& address, const TString& host, ui16 port) + : TNodeInfo() + { + Address = address; + Host = host; + ResolveHost = host; + Port = port; + } + + TNodeInfo(const TString& address, + const TString& host, + const TString& resolveHost, + ui16 port, const TNodeLocation& location) - : TNodeInfo() - { - Address = address; - Host = host; - ResolveHost = resolveHost; - Port = port; - Location = location; - } - - // for testing purposes only - TNodeInfo& operator=(const std::pair<TString, ui32>& pr) { - Address = pr.first; - Host = pr.first; - ResolveHost = pr.first; - Port = pr.second; - return *this; - } - - TNodeInfo& operator=(const TNodeInfo& ni) { - Address = ni.Address; - Host = ni.Host; - ResolveHost = ni.ResolveHost; - Port = ni.Port; - Location = ni.Location; - return *this; - } - }; - - TMap<ui32, TNodeInfo> StaticNodeTable; - - bool IsEntriesUnique() const; + : TNodeInfo() + { + Address = address; + Host = host; + ResolveHost = resolveHost; + Port = port; + Location = location; + } + + // for testing purposes only + TNodeInfo& operator=(const std::pair<TString, ui32>& pr) { + Address = pr.first; + Host = pr.first; + ResolveHost = pr.first; + Port = pr.second; + return *this; + } + + TNodeInfo& operator=(const TNodeInfo& ni) { + Address = ni.Address; + Host = ni.Host; + ResolveHost = ni.ResolveHost; + Port = ni.Port; + Location = ni.Location; + return *this; + } + }; + + TMap<ui32, TNodeInfo> StaticNodeTable; + + bool IsEntriesUnique() const; }; - struct TNodeRegistrarSetup { + struct TNodeRegistrarSetup { TActorId ServiceID; - TIntrusivePtr<TInterconnectGlobalState> GlobalState; - }; + TIntrusivePtr<TInterconnectGlobalState> GlobalState; + }; TActorId GetNameserviceActorId(); @@ -129,9 +129,9 @@ namespace NActors { * Const table-lookup based name service */ - IActor* CreateNameserverTable( - const TIntrusivePtr<TTableNameserverSetup>& setup, - ui32 poolId = 0); + IActor* CreateNameserverTable( + const TIntrusivePtr<TTableNameserverSetup>& setup, + ui32 poolId = 0); /** * Name service which can be paired with external discovery service. diff --git a/library/cpp/actors/interconnect/interconnect_address.cpp b/library/cpp/actors/interconnect/interconnect_address.cpp index 8f474f5a39..8dff6ecb42 100644 --- a/library/cpp/actors/interconnect/interconnect_address.cpp +++ b/library/cpp/actors/interconnect/interconnect_address.cpp @@ -9,9 +9,9 @@ #endif namespace NInterconnect { - TAddress::TAddress() { + TAddress::TAddress() { memset(&Addr, 0, sizeof(Addr)); - } + } TAddress::TAddress(NAddr::IRemoteAddr& addr) { socklen_t len = addr.Len(); @@ -19,42 +19,42 @@ namespace NInterconnect { memcpy(&Addr.Generic, addr.Addr(), len); } - int TAddress::GetFamily() const { + int TAddress::GetFamily() const { return Addr.Generic.sa_family; - } + } - socklen_t TAddress::Size() const { + socklen_t TAddress::Size() const { switch (Addr.Generic.sa_family) { - case AF_INET6: + case AF_INET6: return sizeof(sockaddr_in6); - case AF_INET: + case AF_INET: return sizeof(sockaddr_in); - default: - return 0; - } - } + default: + return 0; + } + } - sockaddr* TAddress::SockAddr() { + sockaddr* TAddress::SockAddr() { return &Addr.Generic; } - const sockaddr* TAddress::SockAddr() const { + const sockaddr* TAddress::SockAddr() const { return &Addr.Generic; - } + } - ui16 TAddress::GetPort() const { + ui16 TAddress::GetPort() const { switch (Addr.Generic.sa_family) { - case AF_INET6: + case AF_INET6: return ntohs(Addr.Ipv6.sin6_port); - case AF_INET: + case AF_INET: return ntohs(Addr.Ipv4.sin_port); - default: - return 0; - } - } + default: + return 0; + } + } - TString TAddress::ToString() const { - return GetAddress() + ":" + ::ToString(GetPort()); + TString TAddress::ToString() const { + return GetAddress() + ":" + ::ToString(GetPort()); } TAddress::TAddress(const char* addr, ui16 port) { @@ -63,8 +63,8 @@ namespace NInterconnect { Addr.Ipv6.sin6_port = htons(port); } else if (inet_pton(Addr.Ipv4.sin_family = AF_INET, addr, &Addr.Ipv4.sin_addr)) { Addr.Ipv4.sin_port = htons(port); - } - } + } + } TAddress::TAddress(const TString& addr, ui16 port) : TAddress(addr.data(), port) @@ -75,17 +75,17 @@ namespace NInterconnect { socklen_t size; switch (Addr.Generic.sa_family) { - case AF_INET6: + case AF_INET6: std::tie(src, size) = std::make_tuple(&Addr.Ipv6.sin6_addr, INET6_ADDRSTRLEN); break; - case AF_INET: + case AF_INET: std::tie(src, size) = std::make_tuple(&Addr.Ipv4.sin_addr, INET_ADDRSTRLEN); break; - default: - return TString(); - } + default: + return TString(); + } char *buffer = static_cast<char*>(alloca(size)); const char *p = inet_ntop(Addr.Generic.sa_family, const_cast<void*>(src), buffer, size); diff --git a/library/cpp/actors/interconnect/interconnect_address.h b/library/cpp/actors/interconnect/interconnect_address.h index e9e0faec81..9c01381ce3 100644 --- a/library/cpp/actors/interconnect/interconnect_address.h +++ b/library/cpp/actors/interconnect/interconnect_address.h @@ -6,24 +6,24 @@ #include <util/generic/string.h> namespace NInterconnect { - class TAddress { + class TAddress { union { sockaddr Generic; sockaddr_in Ipv4; sockaddr_in6 Ipv6; } Addr; - public: - TAddress(); - TAddress(const char* addr, ui16 port); + public: + TAddress(); + TAddress(const char* addr, ui16 port); TAddress(const TString& addr, ui16 port); TAddress(NAddr::IRemoteAddr& addr); - int GetFamily() const; - socklen_t Size() const; - ::sockaddr* SockAddr(); - const ::sockaddr* SockAddr() const; - ui16 GetPort() const; - TString GetAddress() const; - TString ToString() const; + int GetFamily() const; + socklen_t Size() const; + ::sockaddr* SockAddr(); + const ::sockaddr* SockAddr() const; + ui16 GetPort() const; + TString GetAddress() const; + TString ToString() const; }; } diff --git a/library/cpp/actors/interconnect/interconnect_channel.cpp b/library/cpp/actors/interconnect/interconnect_channel.cpp index a66ba2a154..bb54667af6 100644 --- a/library/cpp/actors/interconnect/interconnect_channel.cpp +++ b/library/cpp/actors/interconnect/interconnect_channel.cpp @@ -11,11 +11,11 @@ LWTRACE_USING(ACTORLIB_PROVIDER); namespace NActors { - DECLARE_WILSON_EVENT(EventSentToSocket); - DECLARE_WILSON_EVENT(EventReceivedFromSocket); + DECLARE_WILSON_EVENT(EventSentToSocket); + DECLARE_WILSON_EVENT(EventReceivedFromSocket); bool TEventOutputChannel::FeedDescriptor(TTcpPacketOutTask& task, TEventHolder& event, ui64 *weightConsumed) { - const size_t amount = sizeof(TChannelPart) + sizeof(TEventDescr); + const size_t amount = sizeof(TChannelPart) + sizeof(TEventDescr); if (task.GetVirtualFreeAmount() < amount) { return false; } @@ -32,8 +32,8 @@ namespace NActors { (ExtendedFormat ? IEventHandle::FlagExtendedFormat : 0); TChannelPart *part = static_cast<TChannelPart*>(task.GetFreeArea()); - part->Channel = ChannelId | TChannelPart::LastPartFlag; - part->Size = sizeof(TEventDescr); + part->Channel = ChannelId | TChannelPart::LastPartFlag; + part->Size = sizeof(TEventDescr); memcpy(part + 1, &event.Descr, sizeof(TEventDescr)); task.AppendBuf(part, amount); *weightConsumed += amount; @@ -41,13 +41,13 @@ namespace NActors { Metrics->UpdateOutputChannelEvents(ChannelId); return true; - } + } void TEventOutputChannel::DropConfirmed(ui64 confirm) { LOG_DEBUG_IC_SESSION("ICOCH98", "Dropping confirmed messages"); for (auto it = NotYetConfirmed.begin(); it != NotYetConfirmed.end() && it->Serial <= confirm; ) { Pool.Release(NotYetConfirmed, it++); - } + } } bool TEventOutputChannel::FeedBuf(TTcpPacketOutTask& task, ui64 serial, ui64 *weightConsumed) { @@ -147,7 +147,7 @@ namespace NActors { NotYetConfirmed.splice(NotYetConfirmed.end(), Queue, Queue.begin()); // move event to not-yet-confirmed queue State = EState::INITIAL; return true; // we have processed whole event, signal to the caller - } + } } } @@ -169,7 +169,7 @@ namespace NActors { Pool.Release(NotYetConfirmed); for (auto& item : Queue) { item.ForwardOnNondelivery(false); - } + } Pool.Release(Queue); } diff --git a/library/cpp/actors/interconnect/interconnect_channel.h b/library/cpp/actors/interconnect/interconnect_channel.h index e4a0ae3cda..e4a4c89ba5 100644 --- a/library/cpp/actors/interconnect/interconnect_channel.h +++ b/library/cpp/actors/interconnect/interconnect_channel.h @@ -18,18 +18,18 @@ namespace NActors { #pragma pack(push, 1) - struct TChannelPart { - ui16 Channel; - ui16 Size; + struct TChannelPart { + ui16 Channel; + ui16 Size; - static constexpr ui16 LastPartFlag = ui16(1) << 15; + static constexpr ui16 LastPartFlag = ui16(1) << 15; TString ToString() const { return TStringBuilder() << "{Channel# " << (Channel & ~LastPartFlag) << " LastPartFlag# " << ((Channel & LastPartFlag) ? "true" : "false") << " Size# " << Size << "}"; } - }; + }; #pragma pack(pop) struct TExSerializedEventTooLarge : std::exception { @@ -41,13 +41,13 @@ namespace NActors { }; class TEventOutputChannel : public TInterconnectLoggingBase { - public: + public: TEventOutputChannel(TEventHolderPool& pool, ui16 id, ui32 peerNodeId, ui32 maxSerializedEventSize, std::shared_ptr<IInterconnectMetrics> metrics, TSessionParams params) : TInterconnectLoggingBase(Sprintf("OutputChannel %" PRIu16 " [node %" PRIu32 "]", id, peerNodeId)) - , Pool(pool) + , Pool(pool) , PeerNodeId(peerNodeId) - , ChannelId(id) + , ChannelId(id) , Metrics(std::move(metrics)) , Params(std::move(params)) , MaxSerializedEventSize(maxSerializedEventSize) @@ -61,33 +61,33 @@ namespace NActors { const ui32 bytes = event.Fill(ev) + sizeof(TEventDescr); OutputQueueSize += bytes; return std::make_pair(bytes, &event); - } + } void DropConfirmed(ui64 confirm); bool FeedBuf(TTcpPacketOutTask& task, ui64 serial, ui64 *weightConsumed); - bool IsEmpty() const { + bool IsEmpty() const { return Queue.empty(); - } + } bool IsWorking() const { return !IsEmpty(); } - ui32 GetQueueSize() const { - return (ui32)Queue.size(); - } + ui32 GetQueueSize() const { + return (ui32)Queue.size(); + } ui64 GetBufferedAmountOfData() const { - return OutputQueueSize; - } + return OutputQueueSize; + } void NotifyUndelivered(); - TEventHolderPool& Pool; + TEventHolderPool& Pool; const ui32 PeerNodeId; - const ui16 ChannelId; + const ui16 ChannelId; std::shared_ptr<IInterconnectMetrics> Metrics; const TSessionParams Params; const ui32 MaxSerializedEventSize; @@ -105,7 +105,7 @@ namespace NActors { static constexpr ui16 MinimumFreeSpace = sizeof(TChannelPart) + sizeof(TEventDescr); - protected: + protected: ui64 OutputQueueSize = 0; std::list<TEventHolder> Queue; @@ -120,8 +120,8 @@ namespace NActors { if (const ui64 amount = std::exchange(UnaccountedTraffic, 0)) { Metrics->UpdateOutputChannelTraffic(ChannelId, amount); } - } + } - friend class TInterconnectSessionTCP; - }; + friend class TInterconnectSessionTCP; + }; } diff --git a/library/cpp/actors/interconnect/interconnect_common.h b/library/cpp/actors/interconnect/interconnect_common.h index 285709a00c..28c40f46ff 100644 --- a/library/cpp/actors/interconnect/interconnect_common.h +++ b/library/cpp/actors/interconnect/interconnect_common.h @@ -23,17 +23,17 @@ namespace NActors { }; struct TInterconnectSettings { - TDuration Handshake; - TDuration DeadPeer; - TDuration CloseOnIdle; - ui32 SendBufferDieLimitInMB = 0; - ui64 OutputBuffersTotalSizeLimitInMB = 0; - ui32 TotalInflightAmountOfData = 0; - bool MergePerPeerCounters = false; + TDuration Handshake; + TDuration DeadPeer; + TDuration CloseOnIdle; + ui32 SendBufferDieLimitInMB = 0; + ui64 OutputBuffersTotalSizeLimitInMB = 0; + ui32 TotalInflightAmountOfData = 0; + bool MergePerPeerCounters = false; bool MergePerDataCenterCounters = false; - ui32 TCPSocketBufferSize = 0; - TDuration PingPeriod = TDuration::Seconds(3); - TDuration ForceConfirmPeriod = TDuration::Seconds(1); + ui32 TCPSocketBufferSize = 0; + TDuration PingPeriod = TDuration::Seconds(3); + TDuration ForceConfirmPeriod = TDuration::Seconds(1); TDuration LostConnection; TDuration BatchPeriod; bool BindOnAllAddresses = true; @@ -54,40 +54,40 @@ namespace NActors { } return res; } - }; + }; - struct TChannelSettings { - ui16 Weight; - }; + struct TChannelSettings { + ui16 Weight; + }; - typedef TMap<ui16, TChannelSettings> TChannelsConfig; + typedef TMap<ui16, TChannelSettings> TChannelsConfig; - using TRegisterMonPageCallback = std::function<void(const TString& path, const TString& title, + using TRegisterMonPageCallback = std::function<void(const TString& path, const TString& title, TActorSystem* actorSystem, const TActorId& actorId)>; using TInitWhiteboardCallback = std::function<void(ui16 icPort, TActorSystem* actorSystem)>; using TUpdateWhiteboardCallback = std::function<void(const TString& peer, bool connected, bool green, bool yellow, - bool orange, bool red, TActorSystem* actorSystem)>; + bool orange, bool red, TActorSystem* actorSystem)>; - struct TInterconnectProxyCommon : TAtomicRefCount<TInterconnectProxyCommon> { + struct TInterconnectProxyCommon : TAtomicRefCount<TInterconnectProxyCommon> { TActorId NameserviceId; - NMonitoring::TDynamicCounterPtr MonCounters; + NMonitoring::TDynamicCounterPtr MonCounters; std::shared_ptr<NMonitoring::IMetricRegistry> Metrics; TChannelsConfig ChannelsConfig; TInterconnectSettings Settings; - TRegisterMonPageCallback RegisterMonPage; + TRegisterMonPageCallback RegisterMonPage; TActorId DestructorId; std::shared_ptr<std::atomic<TAtomicBase>> DestructorQueueSize; TAtomicBase MaxDestructorQueueSize = 1024 * 1024 * 1024; - TString ClusterUUID; - TVector<TString> AcceptUUID; + TString ClusterUUID; + TVector<TString> AcceptUUID; ui64 StartTime = GetCycleCountFast(); - TString TechnicalSelfHostName; + TString TechnicalSelfHostName; TInitWhiteboardCallback InitWhiteboard; - TUpdateWhiteboardCallback UpdateWhiteboard; - ui32 HandshakeBallastSize = 0; - TAtomic StartedSessionKiller = 0; + TUpdateWhiteboardCallback UpdateWhiteboard; + ui32 HandshakeBallastSize = 0; + TAtomic StartedSessionKiller = 0; TScopeId LocalScopeId; std::shared_ptr<TEventFilter> EventFilter; TString Cookie; // unique random identifier of a node instance (generated randomly at every start) @@ -101,6 +101,6 @@ namespace NActors { TMaybe<TVersionInfo> VersionInfo; using TPtr = TIntrusivePtr<TInterconnectProxyCommon>; - }; + }; -} +} diff --git a/library/cpp/actors/interconnect/interconnect_handshake.cpp b/library/cpp/actors/interconnect/interconnect_handshake.cpp index 9ede998d8e..705d6f90af 100644 --- a/library/cpp/actors/interconnect/interconnect_handshake.cpp +++ b/library/cpp/actors/interconnect/interconnect_handshake.cpp @@ -62,15 +62,15 @@ namespace NActors { ui32 Checksum; ui32 Size; - ui32 CalculateChecksum(const void* data, size_t len) const { + ui32 CalculateChecksum(const void* data, size_t len) const { return Crc32cExtendMSanCompatible(Crc32cExtendMSanCompatible(0, &Size, sizeof(Size)), data, len); } - void Sign(const void* data, size_t len) { + void Sign(const void* data, size_t len) { Checksum = CalculateChecksum(data, len); } - bool Check(const void* data, size_t len) const { + bool Check(const void* data, size_t len) const { return Checksum == CalculateChecksum(data, len); } }; @@ -474,7 +474,7 @@ namespace NActors { if (const ui32 size = Common->HandshakeBallastSize) { TString ballast(size, 0); - char* data = ballast.Detach(); + char* data = ballast.Detach(); for (ui32 i = 0; i < size; ++i) { data[i] = i; } @@ -721,7 +721,7 @@ namespace NActors { } } - template <typename T> + template <typename T> void SendExBlock(const T& proto, const char* what) { TString data; Y_PROTOBUF_SUPPRESS_NODISCARD proto.SerializeToString(&data); @@ -760,7 +760,7 @@ namespace NActors { Send(GetActorSystem()->InterconnectProxy(PeerNodeId), ev.Release()); } - template <typename TEvent> + template <typename TEvent> THolder<typename TEvent::THandle> WaitForSpecificEvent(TString state, TInstant deadline = TInstant::Max()) { State = std::move(state); return TActorCoroImpl::WaitForSpecificEvent<TEvent>(deadline); @@ -778,7 +778,7 @@ namespace NActors { return WaitForSpecificEvent<TEvent>(std::move(state)); } - template <typename T1, typename T2, typename... TOther> + template <typename T1, typename T2, typename... TOther> THolder<IEventHandle> AskProxy(THolder<IEventBase> ev, TString state) { SendToProxy(std::move(ev)); return WaitForSpecificEvent<T1, T2, TOther...>(std::move(state)); @@ -930,7 +930,7 @@ namespace NActors { template <typename TDataPtr, typename TSendRecvFunc> void Process(TDataPtr buffer, size_t len, TSendRecvFunc&& sendRecv, bool read, bool write, TString state) { Y_VERIFY(Socket); - NInterconnect::TStreamSocket* sock = Socket.Get(); + NInterconnect::TStreamSocket* sock = Socket.Get(); ssize_t (NInterconnect::TStreamSocket::*pfn)(TDataPtr, size_t, TString*) const = sendRecv; size_t processed = 0; @@ -971,7 +971,7 @@ namespace NActors { return std::move(response->Get()->Node); } - template <typename T> + template <typename T> static THolder<TProgramInfo> GetProgramInfo(const T& proto) { auto programInfo = MakeHolder<TProgramInfo>(); programInfo->PID = proto.GetProgramPID(); diff --git a/library/cpp/actors/interconnect/interconnect_handshake.h b/library/cpp/actors/interconnect/interconnect_handshake.h index b3c0db6c5d..1b957570ad 100644 --- a/library/cpp/actors/interconnect/interconnect_handshake.h +++ b/library/cpp/actors/interconnect/interconnect_handshake.h @@ -10,10 +10,10 @@ #include "events_local.h" namespace NActors { - static constexpr TDuration DEFAULT_HANDSHAKE_TIMEOUT = TDuration::Seconds(1); - static constexpr ui64 INTERCONNECT_PROTOCOL_VERSION = 2; + static constexpr TDuration DEFAULT_HANDSHAKE_TIMEOUT = TDuration::Seconds(1); + static constexpr ui64 INTERCONNECT_PROTOCOL_VERSION = 2; - using TSocketPtr = TIntrusivePtr<NInterconnect::TStreamSocket>; + using TSocketPtr = TIntrusivePtr<NInterconnect::TStreamSocket>; IActor* CreateOutgoingHandshakeActor(TInterconnectProxyCommon::TPtr common, const TActorId& self, const TActorId& peer, ui32 nodeId, ui64 nextPacket, TString peerHostName, diff --git a/library/cpp/actors/interconnect/interconnect_impl.h b/library/cpp/actors/interconnect/interconnect_impl.h index ee29e4d397..716bef412e 100644 --- a/library/cpp/actors/interconnect/interconnect_impl.h +++ b/library/cpp/actors/interconnect/interconnect_impl.h @@ -1,5 +1,5 @@ #pragma once - + #include "interconnect.h" #include <library/cpp/actors/protos/interconnect.pb.h> #include <library/cpp/actors/core/event_pb.h> @@ -7,39 +7,39 @@ #include <library/cpp/monlib/dynamic_counters/counters.h> namespace NActors { - // resolve node info - struct TEvInterconnect::TEvResolveNode: public TEventPB<TEvInterconnect::TEvResolveNode, NActorsInterconnect::TEvResolveNode, TEvInterconnect::EvResolveNode> { - TEvResolveNode() { - } + // resolve node info + struct TEvInterconnect::TEvResolveNode: public TEventPB<TEvInterconnect::TEvResolveNode, NActorsInterconnect::TEvResolveNode, TEvInterconnect::EvResolveNode> { + TEvResolveNode() { + } TEvResolveNode(ui32 nodeId, TInstant deadline = TInstant::Max()) { - Record.SetNodeId(nodeId); + Record.SetNodeId(nodeId); if (deadline != TInstant::Max()) { Record.SetDeadline(deadline.GetValue()); } - } - }; - - // node info - struct TEvInterconnect::TEvNodeAddress: public TEventPB<TEvInterconnect::TEvNodeAddress, NActorsInterconnect::TEvNodeInfo, TEvInterconnect::EvNodeAddress> { - TEvNodeAddress() { - } - - TEvNodeAddress(ui32 nodeId) { - Record.SetNodeId(nodeId); - } - }; - - // register node - struct TEvInterconnect::TEvRegisterNode: public TEventBase<TEvInterconnect::TEvRegisterNode, TEvInterconnect::EvRegisterNode> { - }; - - // reply on register node - struct TEvInterconnect::TEvRegisterNodeResult: public TEventBase<TEvInterconnect::TEvRegisterNodeResult, TEvInterconnect::EvRegisterNodeResult> { - }; - - // disconnect - struct TEvInterconnect::TEvDisconnect: public TEventLocal<TEvInterconnect::TEvDisconnect, TEvInterconnect::EvDisconnect> { - }; + } + }; + + // node info + struct TEvInterconnect::TEvNodeAddress: public TEventPB<TEvInterconnect::TEvNodeAddress, NActorsInterconnect::TEvNodeInfo, TEvInterconnect::EvNodeAddress> { + TEvNodeAddress() { + } + + TEvNodeAddress(ui32 nodeId) { + Record.SetNodeId(nodeId); + } + }; + + // register node + struct TEvInterconnect::TEvRegisterNode: public TEventBase<TEvInterconnect::TEvRegisterNode, TEvInterconnect::EvRegisterNode> { + }; + + // reply on register node + struct TEvInterconnect::TEvRegisterNodeResult: public TEventBase<TEvInterconnect::TEvRegisterNodeResult, TEvInterconnect::EvRegisterNodeResult> { + }; + + // disconnect + struct TEvInterconnect::TEvDisconnect: public TEventLocal<TEvInterconnect::TEvDisconnect, TEvInterconnect::EvDisconnect> { + }; } diff --git a/library/cpp/actors/interconnect/interconnect_nameserver_table.cpp b/library/cpp/actors/interconnect/interconnect_nameserver_table.cpp index 43419bf70d..a2071bbc80 100644 --- a/library/cpp/actors/interconnect/interconnect_nameserver_table.cpp +++ b/library/cpp/actors/interconnect/interconnect_nameserver_table.cpp @@ -10,57 +10,57 @@ namespace NActors { class TInterconnectNameserverTable: public TInterconnectNameserverBase<TInterconnectNameserverTable> { - TIntrusivePtr<TTableNameserverSetup> Config; + TIntrusivePtr<TTableNameserverSetup> Config; - public: - static constexpr EActivityType ActorActivityType() { + public: + static constexpr EActivityType ActorActivityType() { return NAMESERVICE; - } + } TInterconnectNameserverTable(const TIntrusivePtr<TTableNameserverSetup>& setup, ui32 /*resolvePoolId*/) : TInterconnectNameserverBase<TInterconnectNameserverTable>(&TInterconnectNameserverTable::StateFunc, setup->StaticNodeTable) - , Config(setup) - { - Y_VERIFY(Config->IsEntriesUnique()); - } + , Config(setup) + { + Y_VERIFY(Config->IsEntriesUnique()); + } - STFUNC(StateFunc) { - try { + STFUNC(StateFunc) { + try { switch (ev->GetTypeRewrite()) { HFunc(TEvInterconnect::TEvResolveNode, Handle); HFunc(TEvResolveAddress, Handle); HFunc(TEvInterconnect::TEvListNodes, Handle); HFunc(TEvInterconnect::TEvGetNode, Handle); - } - } catch (...) { - // on error - do nothing + } + } catch (...) { + // on error - do nothing } } - }; + }; IActor* CreateNameserverTable(const TIntrusivePtr<TTableNameserverSetup>& setup, ui32 poolId) { - return new TInterconnectNameserverTable(setup, poolId); + return new TInterconnectNameserverTable(setup, poolId); } - bool TTableNameserverSetup::IsEntriesUnique() const { - TVector<const TNodeInfo*> infos; - infos.reserve(StaticNodeTable.size()); - for (const auto& x : StaticNodeTable) - infos.push_back(&x.second); + bool TTableNameserverSetup::IsEntriesUnique() const { + TVector<const TNodeInfo*> infos; + infos.reserve(StaticNodeTable.size()); + for (const auto& x : StaticNodeTable) + infos.push_back(&x.second); auto CompareAddressLambda = - [](const TNodeInfo* left, const TNodeInfo* right) { - return left->Port == right->Port ? left->Address < right->Address : left->Port < right->Port; - }; + [](const TNodeInfo* left, const TNodeInfo* right) { + return left->Port == right->Port ? left->Address < right->Address : left->Port < right->Port; + }; Sort(infos, CompareAddressLambda); - for (ui32 idx = 1, end = StaticNodeTable.size(); idx < end; ++idx) { - const TNodeInfo* left = infos[idx - 1]; - const TNodeInfo* right = infos[idx]; + for (ui32 idx = 1, end = StaticNodeTable.size(); idx < end; ++idx) { + const TNodeInfo* left = infos[idx - 1]; + const TNodeInfo* right = infos[idx]; if (left->Address && left->Address == right->Address && left->Port == right->Port) - return false; - } + return false; + } auto CompareHostLambda = [](const TNodeInfo* left, const TNodeInfo* right) { @@ -76,8 +76,8 @@ namespace NActors { return false; } - return true; - } + return true; + } TActorId GetNameserviceActorId() { return TActorId(0, "namesvc"); diff --git a/library/cpp/actors/interconnect/interconnect_stream.cpp b/library/cpp/actors/interconnect/interconnect_stream.cpp index 158ebc9e1d..52e4491cec 100644 --- a/library/cpp/actors/interconnect/interconnect_stream.cpp +++ b/library/cpp/actors/interconnect/interconnect_stream.cpp @@ -23,74 +23,74 @@ #include <cerrno> namespace NInterconnect { - namespace { - inline int - LastSocketError() { + namespace { + inline int + LastSocketError() { #if defined(_win_) - return WSAGetLastError(); + return WSAGetLastError(); #else - return errno; + return errno; #endif - } - } - - TSocket::TSocket(SOCKET fd) - : Descriptor(fd) - { - } - - TSocket::~TSocket() { - if (Descriptor == INVALID_SOCKET) { - return; - } - - auto const result = ::closesocket(Descriptor); - if (result == 0) - return; - switch (LastSocketError()) { - case EBADF: - Y_FAIL("Close bad descriptor"); - case EINTR: - break; - case EIO: - Y_FAIL("EIO"); - default: - Y_FAIL("It's something unexpected"); - } - } - - int TSocket::GetDescriptor() { - return Descriptor; - } - - int - TSocket::Bind(const TAddress& addr) const { - const auto ret = ::bind(Descriptor, addr.SockAddr(), addr.Size()); - if (ret < 0) - return -LastSocketError(); - - return 0; - } - - int - TSocket::Shutdown(int how) const { - const auto ret = ::shutdown(Descriptor, how); - if (ret < 0) - return -LastSocketError(); - - return 0; - } - - int TSocket::GetConnectStatus() const { - int err = 0; - socklen_t len = sizeof(err); - if (getsockopt(Descriptor, SOL_SOCKET, SO_ERROR, reinterpret_cast<char*>(&err), &len) == -1) { - err = LastSocketError(); - } - return err; - } - - ///////////////////////////////////////////////////////////////// + } + } + + TSocket::TSocket(SOCKET fd) + : Descriptor(fd) + { + } + + TSocket::~TSocket() { + if (Descriptor == INVALID_SOCKET) { + return; + } + + auto const result = ::closesocket(Descriptor); + if (result == 0) + return; + switch (LastSocketError()) { + case EBADF: + Y_FAIL("Close bad descriptor"); + case EINTR: + break; + case EIO: + Y_FAIL("EIO"); + default: + Y_FAIL("It's something unexpected"); + } + } + + int TSocket::GetDescriptor() { + return Descriptor; + } + + int + TSocket::Bind(const TAddress& addr) const { + const auto ret = ::bind(Descriptor, addr.SockAddr(), addr.Size()); + if (ret < 0) + return -LastSocketError(); + + return 0; + } + + int + TSocket::Shutdown(int how) const { + const auto ret = ::shutdown(Descriptor, how); + if (ret < 0) + return -LastSocketError(); + + return 0; + } + + int TSocket::GetConnectStatus() const { + int err = 0; + socklen_t len = sizeof(err); + if (getsockopt(Descriptor, SOL_SOCKET, SO_ERROR, reinterpret_cast<char*>(&err), &len) == -1) { + err = LastSocketError(); + } + return err; + } + + ///////////////////////////////////////////////////////////////// TIntrusivePtr<TStreamSocket> TStreamSocket::Make(int domain) { const SOCKET res = ::socket(domain, SOCK_STREAM | SOCK_NONBLOCK, 0); @@ -99,106 +99,106 @@ namespace NInterconnect { Y_VERIFY(err != EMFILE && err != ENFILE); } return MakeIntrusive<TStreamSocket>(res); - } + } - TStreamSocket::TStreamSocket(SOCKET fd) - : TSocket(fd) - { - } + TStreamSocket::TStreamSocket(SOCKET fd) + : TSocket(fd) + { + } - ssize_t + ssize_t TStreamSocket::Send(const void* msg, size_t len, TString* /*err*/) const { const auto ret = ::send(Descriptor, static_cast<const char*>(msg), int(len), 0); - if (ret < 0) - return -LastSocketError(); + if (ret < 0) + return -LastSocketError(); - return ret; - } + return ret; + } - ssize_t + ssize_t TStreamSocket::Recv(void* buf, size_t len, TString* /*err*/) const { const auto ret = ::recv(Descriptor, static_cast<char*>(buf), int(len), 0); - if (ret < 0) - return -LastSocketError(); + if (ret < 0) + return -LastSocketError(); - return ret; - } + return ret; + } - ssize_t - TStreamSocket::WriteV(const struct iovec* iov, int iovcnt) const { + ssize_t + TStreamSocket::WriteV(const struct iovec* iov, int iovcnt) const { #ifndef _win_ - const auto ret = ::writev(Descriptor, iov, iovcnt); - if (ret < 0) - return -LastSocketError(); - return ret; + const auto ret = ::writev(Descriptor, iov, iovcnt); + if (ret < 0) + return -LastSocketError(); + return ret; #else Y_FAIL("WriteV() unsupported on Windows"); #endif - } + } - ssize_t - TStreamSocket::ReadV(const struct iovec* iov, int iovcnt) const { + ssize_t + TStreamSocket::ReadV(const struct iovec* iov, int iovcnt) const { #ifndef _win_ - const auto ret = ::readv(Descriptor, iov, iovcnt); - if (ret < 0) - return -LastSocketError(); - return ret; + const auto ret = ::readv(Descriptor, iov, iovcnt); + if (ret < 0) + return -LastSocketError(); + return ret; #else Y_FAIL("ReadV() unsupported on Windows"); #endif - } + } - ssize_t TStreamSocket::GetUnsentQueueSize() const { - int num = -1; + ssize_t TStreamSocket::GetUnsentQueueSize() const { + int num = -1; #ifndef _win_ // we have no means to determine output queue size on Windows - if (ioctl(Descriptor, TIOCOUTQ, &num) == -1) { - num = -1; - } -#endif - return num; - } - - int - TStreamSocket::Connect(const TAddress& addr) const { - const auto ret = ::connect(Descriptor, addr.SockAddr(), addr.Size()); - if (ret < 0) - return -LastSocketError(); - - return ret; - } - - int - TStreamSocket::Connect(const NAddr::IRemoteAddr* addr) const { - const auto ret = ::connect(Descriptor, addr->Addr(), addr->Len()); - if (ret < 0) - return -LastSocketError(); - - return ret; - } - - int - TStreamSocket::Listen(int backlog) const { - const auto ret = ::listen(Descriptor, backlog); - if (ret < 0) - return -LastSocketError(); - - return ret; - } - - int - TStreamSocket::Accept(TAddress& acceptedAddr) const { - socklen_t acceptedSize = sizeof(::sockaddr_in6); - const auto ret = ::accept(Descriptor, acceptedAddr.SockAddr(), &acceptedSize); - if (ret == INVALID_SOCKET) - return -LastSocketError(); - - return ret; - } - - void - TStreamSocket::SetSendBufferSize(i32 len) const { - (void)SetSockOpt(Descriptor, SOL_SOCKET, SO_SNDBUF, len); - } + if (ioctl(Descriptor, TIOCOUTQ, &num) == -1) { + num = -1; + } +#endif + return num; + } + + int + TStreamSocket::Connect(const TAddress& addr) const { + const auto ret = ::connect(Descriptor, addr.SockAddr(), addr.Size()); + if (ret < 0) + return -LastSocketError(); + + return ret; + } + + int + TStreamSocket::Connect(const NAddr::IRemoteAddr* addr) const { + const auto ret = ::connect(Descriptor, addr->Addr(), addr->Len()); + if (ret < 0) + return -LastSocketError(); + + return ret; + } + + int + TStreamSocket::Listen(int backlog) const { + const auto ret = ::listen(Descriptor, backlog); + if (ret < 0) + return -LastSocketError(); + + return ret; + } + + int + TStreamSocket::Accept(TAddress& acceptedAddr) const { + socklen_t acceptedSize = sizeof(::sockaddr_in6); + const auto ret = ::accept(Descriptor, acceptedAddr.SockAddr(), &acceptedSize); + if (ret == INVALID_SOCKET) + return -LastSocketError(); + + return ret; + } + + void + TStreamSocket::SetSendBufferSize(i32 len) const { + (void)SetSockOpt(Descriptor, SOL_SOCKET, SO_SNDBUF, len); + } ui32 TStreamSocket::GetSendBufferSize() const { ui32 res = 0; @@ -206,7 +206,7 @@ namespace NInterconnect { return res; } - ////////////////////////////////////////////////////// + ////////////////////////////////////////////////////// TDatagramSocket::TPtr TDatagramSocket::Make(int domain) { const SOCKET res = ::socket(domain, SOCK_DGRAM, 0); @@ -215,31 +215,31 @@ namespace NInterconnect { Y_VERIFY(err != EMFILE && err != ENFILE); } return std::make_shared<TDatagramSocket>(res); - } - - TDatagramSocket::TDatagramSocket(SOCKET fd) - : TSocket(fd) - { - } - - ssize_t - TDatagramSocket::SendTo(const void* msg, size_t len, const TAddress& toAddr) const { - const auto ret = ::sendto(Descriptor, static_cast<const char*>(msg), int(len), 0, toAddr.SockAddr(), toAddr.Size()); - if (ret < 0) - return -LastSocketError(); - - return ret; - } - - ssize_t - TDatagramSocket::RecvFrom(void* buf, size_t len, TAddress& fromAddr) const { - socklen_t fromSize = sizeof(::sockaddr_in6); - const auto ret = ::recvfrom(Descriptor, static_cast<char*>(buf), int(len), 0, fromAddr.SockAddr(), &fromSize); - if (ret < 0) - return -LastSocketError(); - - return ret; - } + } + + TDatagramSocket::TDatagramSocket(SOCKET fd) + : TSocket(fd) + { + } + + ssize_t + TDatagramSocket::SendTo(const void* msg, size_t len, const TAddress& toAddr) const { + const auto ret = ::sendto(Descriptor, static_cast<const char*>(msg), int(len), 0, toAddr.SockAddr(), toAddr.Size()); + if (ret < 0) + return -LastSocketError(); + + return ret; + } + + ssize_t + TDatagramSocket::RecvFrom(void* buf, size_t len, TAddress& fromAddr) const { + socklen_t fromSize = sizeof(::sockaddr_in6); + const auto ret = ::recvfrom(Descriptor, static_cast<char*>(buf), int(len), 0, fromAddr.SockAddr(), &fromSize); + if (ret < 0) + return -LastSocketError(); + + return ret; + } // deleter for SSL objects diff --git a/library/cpp/actors/interconnect/interconnect_stream.h b/library/cpp/actors/interconnect/interconnect_stream.h index 074adc6e74..af7176b5f0 100644 --- a/library/cpp/actors/interconnect/interconnect_stream.h +++ b/library/cpp/actors/interconnect/interconnect_stream.h @@ -15,15 +15,15 @@ #include <sys/uio.h> namespace NInterconnect { - class TSocket: public NActors::TSharedDescriptor, public TNonCopyable { - protected: - TSocket(SOCKET fd); + class TSocket: public NActors::TSharedDescriptor, public TNonCopyable { + protected: + TSocket(SOCKET fd); - virtual ~TSocket() override; + virtual ~TSocket() override; SOCKET Descriptor; - virtual int GetDescriptor() override; + virtual int GetDescriptor() override; private: friend class TSecureSocket; @@ -32,21 +32,21 @@ namespace NInterconnect { return std::exchange(Descriptor, INVALID_SOCKET); } - public: - operator SOCKET() const { - return Descriptor; - } + public: + operator SOCKET() const { + return Descriptor; + } - int Bind(const TAddress& addr) const; - int Shutdown(int how) const; - int GetConnectStatus() const; - }; + int Bind(const TAddress& addr) const; + int Shutdown(int how) const; + int GetConnectStatus() const; + }; - class TStreamSocket: public TSocket { - public: - TStreamSocket(SOCKET fd); + class TStreamSocket: public TSocket { + public: + TStreamSocket(SOCKET fd); - static TIntrusivePtr<TStreamSocket> Make(int domain); + static TIntrusivePtr<TStreamSocket> Make(int domain); virtual ssize_t Send(const void* msg, size_t len, TString *err = nullptr) const; virtual ssize_t Recv(void* buf, size_t len, TString *err = nullptr) const; @@ -54,16 +54,16 @@ namespace NInterconnect { virtual ssize_t WriteV(const struct iovec* iov, int iovcnt) const; virtual ssize_t ReadV(const struct iovec* iov, int iovcnt) const; - int Connect(const TAddress& addr) const; - int Connect(const NAddr::IRemoteAddr* addr) const; - int Listen(int backlog) const; - int Accept(TAddress& acceptedAddr) const; + int Connect(const TAddress& addr) const; + int Connect(const NAddr::IRemoteAddr* addr) const; + int Listen(int backlog) const; + int Accept(TAddress& acceptedAddr) const; - ssize_t GetUnsentQueueSize() const; + ssize_t GetUnsentQueueSize() const; - void SetSendBufferSize(i32 len) const; + void SetSendBufferSize(i32 len) const; ui32 GetSendBufferSize() const; - }; + }; class TSecureSocketContext { class TImpl; @@ -116,16 +116,16 @@ namespace NInterconnect { bool WantWrite() const; }; - class TDatagramSocket: public TSocket { - public: - typedef std::shared_ptr<TDatagramSocket> TPtr; + class TDatagramSocket: public TSocket { + public: + typedef std::shared_ptr<TDatagramSocket> TPtr; - TDatagramSocket(SOCKET fd); + TDatagramSocket(SOCKET fd); - static TPtr Make(int domain); + static TPtr Make(int domain); - ssize_t SendTo(const void* msg, size_t len, const TAddress& toAddr) const; - ssize_t RecvFrom(void* buf, size_t len, TAddress& fromAddr) const; - }; + ssize_t SendTo(const void* msg, size_t len, const TAddress& toAddr) const; + ssize_t RecvFrom(void* buf, size_t len, TAddress& fromAddr) const; + }; } diff --git a/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp b/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp index 0abe9fe659..19a1ed3f57 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp +++ b/library/cpp/actors/interconnect/interconnect_tcp_input_session.cpp @@ -339,7 +339,7 @@ namespace NActors { PreallocateBuffers(); TStackVec<TIoVec, NumPreallocatedBuffers> buffs; - for (const auto& item : Buffers) { + for (const auto& item : Buffers) { TIoVec iov{item->GetBuffer(), item->GetCapacity()}; buffs.push_back(iov); if (Params.Encryption) { @@ -347,7 +347,7 @@ namespace NActors { } } - const struct iovec* iovec = reinterpret_cast<const struct iovec*>(buffs.data()); + const struct iovec* iovec = reinterpret_cast<const struct iovec*>(buffs.data()); int iovcnt = buffs.size(); ssize_t recvres = 0; @@ -473,4 +473,4 @@ namespace NActors { } -} +} diff --git a/library/cpp/actors/interconnect/interconnect_tcp_proxy.cpp b/library/cpp/actors/interconnect/interconnect_tcp_proxy.cpp index 7e2d8ccb94..65c8ae6fa5 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_proxy.cpp +++ b/library/cpp/actors/interconnect/interconnect_tcp_proxy.cpp @@ -7,17 +7,17 @@ #include <util/system/getpid.h> namespace NActors { - static constexpr TDuration GetNodeRequestTimeout = TDuration::Seconds(5); + static constexpr TDuration GetNodeRequestTimeout = TDuration::Seconds(5); - static constexpr TDuration FirstErrorSleep = TDuration::MilliSeconds(10); - static constexpr TDuration MaxErrorSleep = TDuration::Seconds(10); - static constexpr ui32 SleepRetryMultiplier = 4; + static constexpr TDuration FirstErrorSleep = TDuration::MilliSeconds(10); + static constexpr TDuration MaxErrorSleep = TDuration::Seconds(10); + static constexpr ui32 SleepRetryMultiplier = 4; - static TString PeerNameForHuman(ui32 nodeNum, const TString& longName, ui16 port) { - TStringBuf token; - TStringBuf(longName).NextTok('.', token); - return ToString<ui32>(nodeNum) + ":" + (token.size() > 0 ? TString(token) : longName) + ":" + ToString<ui16>(port); - } + static TString PeerNameForHuman(ui32 nodeNum, const TString& longName, ui16 port) { + TStringBuf token; + TStringBuf(longName).NextTok('.', token); + return ToString<ui32>(nodeNum) + ":" + (token.size() > 0 ? TString(token) : longName) + ":" + ToString<ui16>(port); + } TInterconnectProxyTCP::TInterconnectProxyTCP(const ui32 node, TInterconnectProxyCommon::TPtr common, IActor **dynamicPtr) @@ -27,14 +27,14 @@ namespace NActors { , Common(std::move(common)) , SecureContext(new NInterconnect::TSecureSocketContext(Common->Settings.Certificate, Common->Settings.PrivateKey, Common->Settings.CaFilePath, Common->Settings.CipherList)) - { + { Y_VERIFY(Common); Y_VERIFY(Common->NameserviceId); if (DynamicPtr) { Y_VERIFY(!*DynamicPtr); *DynamicPtr = this; } - } + } void TInterconnectProxyTCP::Bootstrap() { SetPrefix(Sprintf("Proxy %s [node %" PRIu32 "]", SelfId().ToString().data(), PeerNodeId)); @@ -54,20 +54,20 @@ namespace NActors { TString path = Sprintf("peer%04" PRIu32, PeerNodeId); TString title = Sprintf("Peer #%04" PRIu32, PeerNodeId); mon(path, title, sys, SelfId()); - } + } } - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // PendingActivation - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // PendingActivation + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void TInterconnectProxyTCP::RequestNodeInfo(STATEFN_SIG) { ICPROXY_PROFILED; - Y_VERIFY(!IncomingHandshakeActor && !OutgoingHandshakeActor && !PendingIncomingHandshakeEvents && !PendingSessionEvents); + Y_VERIFY(!IncomingHandshakeActor && !OutgoingHandshakeActor && !PendingIncomingHandshakeEvents && !PendingSessionEvents); EnqueueSessionEvent(ev); StartConfiguring(); - } + } void TInterconnectProxyTCP::RequestNodeInfoForIncomingHandshake(STATEFN_SIG) { ICPROXY_PROFILED; @@ -77,37 +77,37 @@ namespace NActors { EnqueueIncomingHandshakeEvent(ev); StartConfiguring(); } - } + } - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // PendingNodeInfo - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // PendingNodeInfo + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void TInterconnectProxyTCP::StartConfiguring() { ICPROXY_PROFILED; - Y_VERIFY(!IncomingHandshakeActor && !OutgoingHandshakeActor); + Y_VERIFY(!IncomingHandshakeActor && !OutgoingHandshakeActor); - // issue node info request + // issue node info request Send(Common->NameserviceId, new TEvInterconnect::TEvGetNode(PeerNodeId)); - // arm configure timer; store pointer to event to ensure that we will handle correct one if there were any other - // wakeup events in flight + // arm configure timer; store pointer to event to ensure that we will handle correct one if there were any other + // wakeup events in flight SwitchToState(__LINE__, "PendingNodeInfo", &TThis::PendingNodeInfo, GetNodeRequestTimeout, - ConfigureTimeoutCookie = new TEvents::TEvWakeup); - } + ConfigureTimeoutCookie = new TEvents::TEvWakeup); + } void TInterconnectProxyTCP::Configure(TEvInterconnect::TEvNodeInfo::TPtr& ev) { ICPROXY_PROFILED; Y_VERIFY(!IncomingHandshakeActor && !OutgoingHandshakeActor && !Session); - if (!ev->Get()->Node) { + if (!ev->Get()->Node) { TransitToErrorState("cannot get node info"); - } else { - auto& info = *ev->Get()->Node; + } else { + auto& info = *ev->Get()->Node; TString name = PeerNameForHuman(PeerNodeId, info.Host, info.Port); - TechnicalPeerHostName = info.Host; + TechnicalPeerHostName = info.Host; if (!Metrics) { Metrics = Common->Metrics ? CreateInterconnectMetrics(Common) : CreateInterconnectCounters(Common); } @@ -122,9 +122,9 @@ namespace NActors { void TInterconnectProxyTCP::ConfigureTimeout(TEvents::TEvWakeup::TPtr& ev) { ICPROXY_PROFILED; - if (ev->Get() == ConfigureTimeoutCookie) { + if (ev->Get() == ConfigureTimeoutCookie) { TransitToErrorState("timed out while waiting for node info"); - } + } } void TInterconnectProxyTCP::ProcessConfigured() { @@ -152,72 +152,72 @@ namespace NActors { void TInterconnectProxyTCP::StartInitialHandshake() { ICPROXY_PROFILED; - // since we are starting initial handshake for some reason, we'll drop any existing handshakes, if any + // since we are starting initial handshake for some reason, we'll drop any existing handshakes, if any DropHandshakes(); - // create and register handshake actor + // create and register handshake actor OutgoingHandshakeActor = Register(CreateOutgoingHandshakeActor(Common, GenerateSessionVirtualId(), TActorId(), PeerNodeId, 0, TechnicalPeerHostName, TSessionParams()), TMailboxType::ReadAsFilled); OutgoingHandshakeActorCreated = TActivationContext::Now(); // prepare for new handshake PrepareNewSessionHandshake(); - } + } void TInterconnectProxyTCP::StartResumeHandshake(ui64 inputCounter) { ICPROXY_PROFILED; - // drop outgoing handshake if we have one; keep incoming handshakes as they may be useful + // drop outgoing handshake if we have one; keep incoming handshakes as they may be useful DropOutgoingHandshake(); - // ensure that we have session - Y_VERIFY(Session); + // ensure that we have session + Y_VERIFY(Session); - // ensure that we have both virtual ids - Y_VERIFY(SessionVirtualId); - Y_VERIFY(RemoteSessionVirtualId); + // ensure that we have both virtual ids + Y_VERIFY(SessionVirtualId); + Y_VERIFY(RemoteSessionVirtualId); - // create and register handshake actor + // create and register handshake actor OutgoingHandshakeActor = Register(CreateOutgoingHandshakeActor(Common, SessionVirtualId, RemoteSessionVirtualId, PeerNodeId, inputCounter, TechnicalPeerHostName, Session->Params), TMailboxType::ReadAsFilled); OutgoingHandshakeActorCreated = TActivationContext::Now(); - } + } void TInterconnectProxyTCP::IssueIncomingHandshakeReply(const TActorId& handshakeId, ui64 peerLocalId, THolder<IEventBase> event) { ICPROXY_PROFILED; - Y_VERIFY(!IncomingHandshakeActor); - IncomingHandshakeActor = handshakeId; + Y_VERIFY(!IncomingHandshakeActor); + IncomingHandshakeActor = handshakeId; IncomingHandshakeActorFilledIn = TActivationContext::Now(); Y_VERIFY(!LastSerialFromIncomingHandshake || *LastSerialFromIncomingHandshake <= peerLocalId); LastSerialFromIncomingHandshake = peerLocalId; if (OutgoingHandshakeActor && SelfId().NodeId() < PeerNodeId) { - // Both outgoing and incoming handshake are in progress. To prevent race condition during semultanous handshake - // incoming handshake must be held till outgoing handshake is complete or failed + // Both outgoing and incoming handshake are in progress. To prevent race condition during semultanous handshake + // incoming handshake must be held till outgoing handshake is complete or failed LOG_DEBUG_IC("ICP06", "reply for incoming handshake (actor %s) is held", IncomingHandshakeActor.ToString().data()); - HeldHandshakeReply = std::move(event); + HeldHandshakeReply = std::move(event); - // Check that we are in one of acceptable states that would properly handle handshake statuses. - const auto state = CurrentStateFunc(); - Y_VERIFY(state == &TThis::PendingConnection || state == &TThis::StateWork, "invalid handshake request in state# %s", State); - } else { - LOG_DEBUG_IC("ICP07", "issued incoming handshake reply"); + // Check that we are in one of acceptable states that would properly handle handshake statuses. + const auto state = CurrentStateFunc(); + Y_VERIFY(state == &TThis::PendingConnection || state == &TThis::StateWork, "invalid handshake request in state# %s", State); + } else { + LOG_DEBUG_IC("ICP07", "issued incoming handshake reply"); - // No race, so we can send reply immediately. - Y_VERIFY(!HeldHandshakeReply); + // No race, so we can send reply immediately. + Y_VERIFY(!HeldHandshakeReply); Send(IncomingHandshakeActor, event.Release()); - // Start waiting for handshake reply, if not yet started; also, if session is already created, then we don't - // switch from working state. - if (!Session) { + // Start waiting for handshake reply, if not yet started; also, if session is already created, then we don't + // switch from working state. + if (!Session) { LOG_INFO_IC("ICP08", "No active sessions, becoming PendingConnection"); SwitchToState(__LINE__, "PendingConnection", &TThis::PendingConnection); - } else { - Y_VERIFY(CurrentStateFunc() == &TThis::StateWork); - } + } else { + Y_VERIFY(CurrentStateFunc() == &TThis::StateWork); + } } } @@ -243,8 +243,8 @@ namespace NActors { } else { // if we already have incoming handshake, then terminate existing one DropIncomingHandshake(); - - // issue reply to the sender, possibly holding it while outgoing handshake is at race + + // issue reply to the sender, possibly holding it while outgoing handshake is at race THolder<IEventBase> reply = IActor::InvokeOtherActor(*Session, &TInterconnectSessionTCP::ProcessHandshakeRequest, ev); return IssueIncomingHandshakeReply(ev->Sender, RemoteSessionVirtualId.LocalId(), std::move(reply)); } @@ -263,27 +263,27 @@ namespace NActors { ui64 remoteStartTime = record.GetProgramStartTime(); ui64 remoteSerial = record.GetSerial(); - if (RemoteProgramInfo && remotePID == RemoteProgramInfo->PID && remoteStartTime == RemoteProgramInfo->StartTime) { - if (remoteSerial < RemoteProgramInfo->Serial) { + if (RemoteProgramInfo && remotePID == RemoteProgramInfo->PID && remoteStartTime == RemoteProgramInfo->StartTime) { + if (remoteSerial < RemoteProgramInfo->Serial) { LOG_INFO_IC("ICP18", "handshake (actor %s) is too old", ev->Sender.ToString().data()); Send(ev->Sender, new TEvents::TEvPoisonPill); - return; - } else { - RemoteProgramInfo->Serial = remoteSerial; - } + return; + } else { + RemoteProgramInfo->Serial = remoteSerial; + } } else { - const auto ptr = new TProgramInfo; - ptr->PID = remotePID; - ptr->StartTime = remoteStartTime; - ptr->Serial = remoteSerial; - RemoteProgramInfo.Reset(ptr); + const auto ptr = new TProgramInfo; + ptr->PID = remotePID; + ptr->StartTime = remoteStartTime; + ptr->Serial = remoteSerial; + RemoteProgramInfo.Reset(ptr); } - /* Let's check peer technical hostname */ + /* Let's check peer technical hostname */ if (record.HasSenderHostName() && TechnicalPeerHostName != record.GetSenderHostName()) { Send(ev->Sender, new TEvHandshakeReplyError("host name mismatch")); - return; - } + return; + } // check sender actor id and check if it is not very old if (LastSerialFromIncomingHandshake) { @@ -303,60 +303,60 @@ namespace NActors { } } - // drop incoming handshake as this is definitely more recent + // drop incoming handshake as this is definitely more recent DropIncomingHandshake(); - // prepare for new session + // prepare for new session PrepareNewSessionHandshake(); - auto event = MakeHolder<TEvHandshakeReplyOK>(); - auto* pb = event->Record.MutableSuccess(); + auto event = MakeHolder<TEvHandshakeReplyOK>(); + auto* pb = event->Record.MutableSuccess(); const TActorId virtualId = GenerateSessionVirtualId(); - pb->SetProtocol(INTERCONNECT_PROTOCOL_VERSION); + pb->SetProtocol(INTERCONNECT_PROTOCOL_VERSION); pb->SetSenderActorId(virtualId.ToString()); - pb->SetProgramPID(GetPID()); + pb->SetProgramPID(GetPID()); pb->SetProgramStartTime(Common->StartTime); - pb->SetSerial(virtualId.LocalId()); + pb->SetSerial(virtualId.LocalId()); IssueIncomingHandshakeReply(ev->Sender, 0, std::move(event)); - } + } void TInterconnectProxyTCP::HandleHandshakeStatus(TEvHandshakeDone::TPtr& ev) { ICPROXY_PROFILED; TEvHandshakeDone *msg = ev->Get(); - // Terminate handshake actor working in opposite direction, if set up. - if (ev->Sender == IncomingHandshakeActor) { - LOG_INFO_IC("ICP19", "incoming handshake succeeded"); + // Terminate handshake actor working in opposite direction, if set up. + if (ev->Sender == IncomingHandshakeActor) { + LOG_INFO_IC("ICP19", "incoming handshake succeeded"); DropIncomingHandshake(false); DropOutgoingHandshake(); - } else if (ev->Sender == OutgoingHandshakeActor) { - LOG_INFO_IC("ICP20", "outgoing handshake succeeded"); + } else if (ev->Sender == OutgoingHandshakeActor) { + LOG_INFO_IC("ICP20", "outgoing handshake succeeded"); DropIncomingHandshake(); DropOutgoingHandshake(false); - } else { + } else { /* It seems to be an old handshake. */ - return; + return; } - Y_VERIFY(!IncomingHandshakeActor && !OutgoingHandshakeActor); + Y_VERIFY(!IncomingHandshakeActor && !OutgoingHandshakeActor); SwitchToState(__LINE__, "StateWork", &TThis::StateWork); - if (Session) { + if (Session) { // this is continuation request, check that virtual ids match Y_VERIFY(SessionVirtualId == msg->Self && RemoteSessionVirtualId == msg->Peer); - } else { + } else { // this is initial request, check that we have virtual ids not filled in Y_VERIFY(!SessionVirtualId && !RemoteSessionVirtualId); - } + } auto error = [&](const char* description) { TransitToErrorState(description); }; - // If session is not created, then create new one. - if (!Session) { + // If session is not created, then create new one. + if (!Session) { RemoteProgramInfo = std::move(msg->ProgramInfo); if (!RemoteProgramInfo) { // we have received resume handshake, but session was closed concurrently while handshaking @@ -374,15 +374,15 @@ namespace NActors { // ensure that we have session local/peer virtual ids Y_VERIFY(Session && SessionVirtualId && RemoteSessionVirtualId); - // Set up new connection for the session. + // Set up new connection for the session. IActor::InvokeOtherActor(*Session, &TInterconnectSessionTCP::SetNewConnection, ev); - // Reset retry timer - HoldByErrorWakeupDuration = TDuration::Zero(); + // Reset retry timer + HoldByErrorWakeupDuration = TDuration::Zero(); - /* Forward all held events */ + /* Forward all held events */ ProcessPendingSessionEvents(); - } + } void TInterconnectProxyTCP::HandleHandshakeStatus(TEvHandshakeFail::TPtr& ev) { ICPROXY_PROFILED; @@ -392,42 +392,42 @@ namespace NActors { (IncomingHandshakeActor && OutgoingHandshakeActor); LogHandshakeFail(ev, inconclusive); - if (ev->Sender == IncomingHandshakeActor) { - LOG_NOTICE_IC("ICP24", "incoming handshake failed, temporary: %" PRIu32 " explanation: %s outgoing: %s", + if (ev->Sender == IncomingHandshakeActor) { + LOG_NOTICE_IC("ICP24", "incoming handshake failed, temporary: %" PRIu32 " explanation: %s outgoing: %s", ui32(ev->Get()->Temporary), ev->Get()->Explanation.data(), OutgoingHandshakeActor.ToString().data()); DropIncomingHandshake(false); - } else if (ev->Sender == OutgoingHandshakeActor) { - LOG_NOTICE_IC("ICP25", "outgoing handshake failed, temporary: %" PRIu32 " explanation: %s incoming: %s held: %s", + } else if (ev->Sender == OutgoingHandshakeActor) { + LOG_NOTICE_IC("ICP25", "outgoing handshake failed, temporary: %" PRIu32 " explanation: %s incoming: %s held: %s", ui32(ev->Get()->Temporary), ev->Get()->Explanation.data(), IncomingHandshakeActor.ToString().data(), - HeldHandshakeReply ? "yes" : "no"); + HeldHandshakeReply ? "yes" : "no"); DropOutgoingHandshake(false); - if (IEventBase* reply = HeldHandshakeReply.Release()) { - Y_VERIFY(IncomingHandshakeActor); + if (IEventBase* reply = HeldHandshakeReply.Release()) { + Y_VERIFY(IncomingHandshakeActor); LOG_DEBUG_IC("ICP26", "sent held handshake reply to %s", IncomingHandshakeActor.ToString().data()); Send(IncomingHandshakeActor, reply); - } + } // if we have no current session, then we have to drop all pending events as the outgoing handshake has failed ProcessPendingSessionEvents(); - } else { - /* It seems to be an old fail, just ignore it */ + } else { + /* It seems to be an old fail, just ignore it */ LOG_NOTICE_IC("ICP27", "obsolete handshake fail ignored"); - return; + return; } if (Metrics) { Metrics->IncHandshakeFails(); - } + } - if (IncomingHandshakeActor || OutgoingHandshakeActor) { - // one of handshakes is still going on - LOG_DEBUG_IC("ICP28", "other handshake is still going on"); - return; - } + if (IncomingHandshakeActor || OutgoingHandshakeActor) { + // one of handshakes is still going on + LOG_DEBUG_IC("ICP28", "other handshake is still going on"); + return; + } - switch (ev->Get()->Temporary) { - case TEvHandshakeFail::HANDSHAKE_FAIL_TRANSIENT: + switch (ev->Get()->Temporary) { + case TEvHandshakeFail::HANDSHAKE_FAIL_TRANSIENT: if (!Session) { if (PendingSessionEvents) { // try to start outgoing handshake as we have some events enqueued @@ -444,22 +444,22 @@ namespace NActors { // we have no active connection in that session, so just restart handshake from last known position IActor::InvokeOtherActor(*Session, &TInterconnectSessionTCP::StartHandshake); } - break; - - case TEvHandshakeFail::HANDSHAKE_FAIL_SESSION_MISMATCH: + break; + + case TEvHandshakeFail::HANDSHAKE_FAIL_SESSION_MISMATCH: StartInitialHandshake(); - break; - - case TEvHandshakeFail::HANDSHAKE_FAIL_PERMANENT: + break; + + case TEvHandshakeFail::HANDSHAKE_FAIL_PERMANENT: TString timeExplanation = " LastSessionDieTime# " + LastSessionDieTime.ToString(); - if (Session) { + if (Session) { InvokeOtherActor(*Session, &TInterconnectSessionTCP::Terminate, TDisconnectReason::HandshakeFailPermanent()); } TransitToErrorState(ev->Get()->Explanation + timeExplanation, false); - break; - } - } + break; + } + } void TInterconnectProxyTCP::LogHandshakeFail(TEvHandshakeFail::TPtr& ev, bool inconclusive) { ICPROXY_PROFILED; @@ -487,69 +487,69 @@ namespace NActors { void TInterconnectProxyTCP::ProcessPendingSessionEvents() { ICPROXY_PROFILED; - while (PendingSessionEvents) { + while (PendingSessionEvents) { TPendingSessionEvent ev = std::move(PendingSessionEvents.front()); PendingSessionEventsSize -= ev.Size; TAutoPtr<IEventHandle> event(ev.Event.Release()); - PendingSessionEvents.pop_front(); + PendingSessionEvents.pop_front(); if (Session) { ForwardSessionEventToSession(event); - } else { + } else { DropSessionEvent(event); } - } + } } void TInterconnectProxyTCP::DropSessionEvent(STATEFN_SIG) { ICPROXY_PROFILED; ValidateEvent(ev, "DropSessionEvent"); - switch (ev->GetTypeRewrite()) { - case TEvInterconnect::EvForward: - if (ev->Flags & IEventHandle::FlagSubscribeOnSession) { + switch (ev->GetTypeRewrite()) { + case TEvInterconnect::EvForward: + if (ev->Flags & IEventHandle::FlagSubscribeOnSession) { Send(ev->Sender, new TEvInterconnect::TEvNodeDisconnected(PeerNodeId), 0, ev->Cookie); - } + } TActivationContext::Send(ev->ForwardOnNondelivery(TEvents::TEvUndelivered::Disconnected)); - break; + break; - case TEvInterconnect::TEvConnectNode::EventType: - case TEvents::TEvSubscribe::EventType: + case TEvInterconnect::TEvConnectNode::EventType: + case TEvents::TEvSubscribe::EventType: Send(ev->Sender, new TEvInterconnect::TEvNodeDisconnected(PeerNodeId), 0, ev->Cookie); - break; + break; - case TEvents::TEvUnsubscribe::EventType: - /* Do nothing */ - break; + case TEvents::TEvUnsubscribe::EventType: + /* Do nothing */ + break; - default: - Y_FAIL("Unexpected type of event in held event queue"); - } + default: + Y_FAIL("Unexpected type of event in held event queue"); + } } void TInterconnectProxyTCP::UnregisterSession(TInterconnectSessionTCP* session) { ICPROXY_PROFILED; - Y_VERIFY(Session && Session == session && SessionID); + Y_VERIFY(Session && Session == session && SessionID); LOG_INFO_IC("ICP30", "unregister session Session# %s VirtualId# %s", SessionID.ToString().data(), SessionVirtualId.ToString().data()); - Session = nullptr; + Session = nullptr; SessionID = TActorId(); - // drop all pending events as we are closed + // drop all pending events as we are closed ProcessPendingSessionEvents(); - // reset virtual ids as this session is terminated + // reset virtual ids as this session is terminated SessionVirtualId = TActorId(); RemoteSessionVirtualId = TActorId(); if (Metrics) { Metrics->IncSessionDeaths(); - } + } LastSessionDieTime = TActivationContext::Now(); - + if (IncomingHandshakeActor || OutgoingHandshakeActor) { PrepareNewSessionHandshake(); } else { @@ -566,22 +566,22 @@ namespace NActors { PendingSessionEvents.emplace_back(TActivationContext::Now() + Common->Settings.MessagePendingTimeout, size, ev); ScheduleCleanupEventQueue(); CleanupEventQueue(); - } + } void TInterconnectProxyTCP::EnqueueIncomingHandshakeEvent(STATEFN_SIG) { ICPROXY_PROFILED; // enqueue handshake request Y_UNUSED(); - PendingIncomingHandshakeEvents.emplace_back(ev); - } + PendingIncomingHandshakeEvents.emplace_back(ev); + } void TInterconnectProxyTCP::EnqueueIncomingHandshakeEvent(TEvHandshakeDone::TPtr& /*ev*/) { ICPROXY_PROFILED; // TEvHandshakeDone can't get into the queue, because we have to process handshake request first; this may be the // race with the previous handshakes, so simply ignore it - } + } void TInterconnectProxyTCP::EnqueueIncomingHandshakeEvent(TEvHandshakeFail::TPtr& ev) { ICPROXY_PROFILED; @@ -599,140 +599,140 @@ namespace NActors { break; } } - } + } void TInterconnectProxyTCP::ForwardSessionEventToSession(STATEFN_SIG) { ICPROXY_PROFILED; - Y_VERIFY(Session && SessionID); + Y_VERIFY(Session && SessionID); ValidateEvent(ev, "ForwardSessionEventToSession"); InvokeOtherActor(*Session, &TInterconnectSessionTCP::Receive, ev, TActivationContext::ActorContextFor(SessionID)); - } + } void TInterconnectProxyTCP::GenerateHttpInfo(NMon::TEvHttpInfo::TPtr& ev) { ICPROXY_PROFILED; LOG_INFO_IC("ICP31", "proxy http called"); - TStringStream str; - - HTML(str) { - DIV_CLASS("panel panel-info") { - DIV_CLASS("panel-heading") { - str << "Proxy"; - } - DIV_CLASS("panel-body") { - TABLE_CLASS("table") { - TABLEHEAD() { - TABLER() { - TABLEH() { - str << "Sensor"; - } - TABLEH() { - str << "Value"; - } - } - } -#define MON_VAR(NAME) \ - TABLER() { \ - TABLED() { \ - str << #NAME; \ - } \ - TABLED() { \ - str << NAME; \ - } \ - } - - TABLEBODY() { + TStringStream str; + + HTML(str) { + DIV_CLASS("panel panel-info") { + DIV_CLASS("panel-heading") { + str << "Proxy"; + } + DIV_CLASS("panel-body") { + TABLE_CLASS("table") { + TABLEHEAD() { + TABLER() { + TABLEH() { + str << "Sensor"; + } + TABLEH() { + str << "Value"; + } + } + } +#define MON_VAR(NAME) \ + TABLER() { \ + TABLED() { \ + str << #NAME; \ + } \ + TABLED() { \ + str << NAME; \ + } \ + } + + TABLEBODY() { MON_VAR(TActivationContext::Now()) - MON_VAR(SessionID) - MON_VAR(LastSessionDieTime) - MON_VAR(IncomingHandshakeActor) - MON_VAR(IncomingHandshakeActorFilledIn) - MON_VAR(IncomingHandshakeActorReset) - MON_VAR(OutgoingHandshakeActor) - MON_VAR(OutgoingHandshakeActorCreated) - MON_VAR(OutgoingHandshakeActorReset) - MON_VAR(State) - MON_VAR(StateSwitchTime) + MON_VAR(SessionID) + MON_VAR(LastSessionDieTime) + MON_VAR(IncomingHandshakeActor) + MON_VAR(IncomingHandshakeActorFilledIn) + MON_VAR(IncomingHandshakeActorReset) + MON_VAR(OutgoingHandshakeActor) + MON_VAR(OutgoingHandshakeActorCreated) + MON_VAR(OutgoingHandshakeActorReset) + MON_VAR(State) + MON_VAR(StateSwitchTime) } } } } - DIV_CLASS("panel panel-info") { - DIV_CLASS("panel-heading") { - str << "Error Log"; - } - DIV_CLASS("panel-body") { - TABLE_CLASS("table") { - TABLEHEAD() { + DIV_CLASS("panel panel-info") { + DIV_CLASS("panel-heading") { + str << "Error Log"; + } + DIV_CLASS("panel-body") { + TABLE_CLASS("table") { + TABLEHEAD() { TABLER() { - TABLEH() { - str << "Timestamp"; + TABLEH() { + str << "Timestamp"; } - TABLEH() { - str << "Elapsed"; + TABLEH() { + str << "Elapsed"; } - TABLEH() { - str << "Kind"; + TABLEH() { + str << "Kind"; } - TABLEH() { - str << "Explanation"; + TABLEH() { + str << "Explanation"; } } } - TABLEBODY() { + TABLEBODY() { const TInstant now = TActivationContext::Now(); - const TInstant barrier = now - TDuration::Minutes(1); - for (auto it = ErrorStateLog.rbegin(); it != ErrorStateLog.rend(); ++it) { - auto wrapper = [&](const auto& lambda) { - if (std::get<0>(*it) > barrier) { - str << "<strong>"; - lambda(); - str << "</strong>"; - } else { - lambda(); - } - }; - TABLER() { - TABLED() { - wrapper([&] { - str << std::get<0>(*it); - }); - } - TABLED() { - wrapper([&] { - str << now - std::get<0>(*it); - }); - } - TABLED() { - wrapper([&] { - str << std::get<1>(*it); - }); - } - TABLED() { - wrapper([&] { - str << std::get<2>(*it); - }); + const TInstant barrier = now - TDuration::Minutes(1); + for (auto it = ErrorStateLog.rbegin(); it != ErrorStateLog.rend(); ++it) { + auto wrapper = [&](const auto& lambda) { + if (std::get<0>(*it) > barrier) { + str << "<strong>"; + lambda(); + str << "</strong>"; + } else { + lambda(); + } + }; + TABLER() { + TABLED() { + wrapper([&] { + str << std::get<0>(*it); + }); + } + TABLED() { + wrapper([&] { + str << now - std::get<0>(*it); + }); + } + TABLED() { + wrapper([&] { + str << std::get<1>(*it); + }); + } + TABLED() { + wrapper([&] { + str << std::get<2>(*it); + }); ui32 rep = std::get<3>(*it); if (rep != 1) { str << " <strong>x" << rep << "</strong>"; } - } - } - } - } + } + } + } + } } } } } - if (Session != nullptr) { + if (Session != nullptr) { Session->GenerateHttpInfo(str); - } - + } + Send(ev->Sender, new NMon::TEvHttpInfoRes(str.Str())); } @@ -746,16 +746,16 @@ namespace NActors { UpdateErrorStateLog(TActivationContext::Now(), "permanent conclusive", explanation); } - Y_VERIFY(Session == nullptr); - Y_VERIFY(!SessionID); + Y_VERIFY(Session == nullptr); + Y_VERIFY(!SessionID); - // recalculate wakeup timeout -- if this is the first failure, then we sleep for default timeout; otherwise we - // sleep N times longer than the previous try, but not longer than desired number of seconds - HoldByErrorWakeupDuration = HoldByErrorWakeupDuration != TDuration::Zero() - ? Min(HoldByErrorWakeupDuration * SleepRetryMultiplier, MaxErrorSleep) - : FirstErrorSleep; + // recalculate wakeup timeout -- if this is the first failure, then we sleep for default timeout; otherwise we + // sleep N times longer than the previous try, but not longer than desired number of seconds + HoldByErrorWakeupDuration = HoldByErrorWakeupDuration != TDuration::Zero() + ? Min(HoldByErrorWakeupDuration * SleepRetryMultiplier, MaxErrorSleep) + : FirstErrorSleep; - // transit to required state and arm wakeup timer + // transit to required state and arm wakeup timer if (Terminated) { // switch to this state permanently SwitchToState(__LINE__, "HoldByError", &TThis::HoldByError); @@ -765,21 +765,21 @@ namespace NActors { HoldByErrorWakeupCookie = new TEvents::TEvWakeup); } - /* Process all pending events. */ + /* Process all pending events. */ ProcessPendingSessionEvents(); - /* Terminate handshakes */ + /* Terminate handshakes */ DropHandshakes(); - /* Terminate pending incoming handshake requests. */ - for (auto& ev : PendingIncomingHandshakeEvents) { + /* Terminate pending incoming handshake requests. */ + for (auto& ev : PendingIncomingHandshakeEvents) { Send(ev->Sender, new TEvents::TEvPoisonPill); if (ev->GetTypeRewrite() == TEvHandshakeFail::EventType) { TEvHandshakeFail::TPtr tmp(static_cast<TEventHandle<TEvHandshakeFail>*>(ev.Release())); LogHandshakeFail(tmp, true); } - } - PendingIncomingHandshakeEvents.clear(); + } + PendingIncomingHandshakeEvents.clear(); } void TInterconnectProxyTCP::WakeupFromErrorState(TEvents::TEvWakeup::TPtr& ev) { @@ -787,39 +787,39 @@ namespace NActors { LOG_INFO_IC("ICP33", "wake up from error state"); - if (ev->Get() == HoldByErrorWakeupCookie) { + if (ev->Get() == HoldByErrorWakeupCookie) { SwitchToInitialState(); - } + } } void TInterconnectProxyTCP::Disconnect() { ICPROXY_PROFILED; - // terminate handshakes (if any) + // terminate handshakes (if any) DropHandshakes(); - if (Session) { + if (Session) { IActor::InvokeOtherActor(*Session, &TInterconnectSessionTCP::Terminate, TDisconnectReason::UserRequest()); - } else { + } else { TransitToErrorState("forced disconnect"); - } + } } void TInterconnectProxyTCP::ScheduleCleanupEventQueue() { ICPROXY_PROFILED; - if (!CleanupEventQueueScheduled && PendingSessionEvents) { + if (!CleanupEventQueueScheduled && PendingSessionEvents) { // apply batching at 50 ms granularity Schedule(Max(TDuration::MilliSeconds(50), PendingSessionEvents.front().Deadline - TActivationContext::Now()), new TEvCleanupEventQueue); - CleanupEventQueueScheduled = true; - } + CleanupEventQueueScheduled = true; + } } void TInterconnectProxyTCP::HandleCleanupEventQueue() { ICPROXY_PROFILED; - Y_VERIFY(CleanupEventQueueScheduled); - CleanupEventQueueScheduled = false; + Y_VERIFY(CleanupEventQueueScheduled); + CleanupEventQueueScheduled = false; CleanupEventQueue(); ScheduleCleanupEventQueue(); } @@ -838,24 +838,24 @@ namespace NActors { } else { break; } - } + } } void TInterconnectProxyTCP::HandleClosePeerSocket() { ICPROXY_PROFILED; - if (Session && Session->Socket) { + if (Session && Session->Socket) { LOG_INFO_IC("ICP34", "closed connection by debug command"); - Session->Socket->Shutdown(SHUT_RDWR); - } + Session->Socket->Shutdown(SHUT_RDWR); + } } void TInterconnectProxyTCP::HandleCloseInputSession() { ICPROXY_PROFILED; - if (Session) { + if (Session) { IActor::InvokeOtherActor(*Session, &TInterconnectSessionTCP::CloseInputSession); - } + } } void TInterconnectProxyTCP::HandlePoisonSession() { @@ -869,11 +869,11 @@ namespace NActors { void TInterconnectProxyTCP::HandleSessionBufferSizeRequest(TEvSessionBufferSizeRequest::TPtr& ev) { ICPROXY_PROFILED; - ui64 bufSize = 0; - if (Session) { - bufSize = Session->TotalOutputQueueSize; - } - + ui64 bufSize = 0; + if (Session) { + bufSize = Session->TotalOutputQueueSize; + } + Send(ev->Sender, new TEvSessionBufferSizeResponse(SessionID, bufSize)); } diff --git a/library/cpp/actors/interconnect/interconnect_tcp_proxy.h b/library/cpp/actors/interconnect/interconnect_tcp_proxy.h index 023e5bd1ee..1f9ce6841c 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_proxy.h +++ b/library/cpp/actors/interconnect/interconnect_tcp_proxy.h @@ -16,22 +16,22 @@ namespace NActors { - /* WARNING: all proxy actors should be alive during actorsystem activity */ + /* WARNING: all proxy actors should be alive during actorsystem activity */ class TInterconnectProxyTCP : public TActor<TInterconnectProxyTCP> , public TInterconnectLoggingBase , public TProfiled { - enum { - EvCleanupEventQueue = EventSpaceBegin(TEvents::ES_PRIVATE), + enum { + EvCleanupEventQueue = EventSpaceBegin(TEvents::ES_PRIVATE), EvQueryStats, EvStats, EvPassAwayIfNeeded, - }; + }; - struct TEvCleanupEventQueue : TEventLocal<TEvCleanupEventQueue, EvCleanupEventQueue> {}; + struct TEvCleanupEventQueue : TEventLocal<TEvCleanupEventQueue, EvCleanupEventQueue> {}; - public: + public: struct TEvQueryStats : TEventLocal<TEvQueryStats, EvQueryStats> {}; struct TProxyStats { @@ -56,9 +56,9 @@ namespace NActors { TProxyStats ProxyStats; }; - static constexpr EActivityType ActorActivityType() { - return INTERCONNECT_PROXY_TCP; - } + static constexpr EActivityType ActorActivityType() { + return INTERCONNECT_PROXY_TCP; + } TInterconnectProxyTCP(const ui32 node, TInterconnectProxyCommon::TPtr common, IActor **dynamicPtr = nullptr); @@ -72,11 +72,11 @@ namespace NActors { void Bootstrap(); void Registered(TActorSystem* sys, const TActorId& owner) override; - private: - friend class TInterconnectSessionTCP; - friend class TInterconnectSessionTCPv0; - friend class THandshake; - friend class TInputSessionTCP; + private: + friend class TInterconnectSessionTCP; + friend class TInterconnectSessionTCPv0; + friend class THandshake; + friend class TInputSessionTCP; void UnregisterSession(TInterconnectSessionTCP* session); @@ -138,46 +138,46 @@ namespace NActors { } \ } - template <typename T> + template <typename T> void Ignore(T& /*ev*/) { ICPROXY_PROFILED; - } + } void Ignore() { ICPROXY_PROFILED; - } + } void Ignore(TEvHandshakeDone::TPtr& ev) { ICPROXY_PROFILED; - Y_VERIFY(ev->Sender != IncomingHandshakeActor); - Y_VERIFY(ev->Sender != OutgoingHandshakeActor); - } + Y_VERIFY(ev->Sender != IncomingHandshakeActor); + Y_VERIFY(ev->Sender != OutgoingHandshakeActor); + } void Ignore(TEvHandshakeFail::TPtr& ev) { ICPROXY_PROFILED; - Y_VERIFY(ev->Sender != IncomingHandshakeActor); - Y_VERIFY(ev->Sender != OutgoingHandshakeActor); + Y_VERIFY(ev->Sender != IncomingHandshakeActor); + Y_VERIFY(ev->Sender != OutgoingHandshakeActor); LogHandshakeFail(ev, true); - } + } - const char* State = nullptr; - TInstant StateSwitchTime; + const char* State = nullptr; + TInstant StateSwitchTime; - template <typename... TArgs> + template <typename... TArgs> void SwitchToState(int line, const char* name, TArgs&&... args) { ICPROXY_PROFILED; LOG_DEBUG_IC("ICP77", "@%d %s -> %s", line, State, name); - State = name; + State = name; StateSwitchTime = TActivationContext::Now(); - Become(std::forward<TArgs>(args)...); + Become(std::forward<TArgs>(args)...); Y_VERIFY(!Terminated || CurrentStateFunc() == &TThis::HoldByError); // ensure we never escape this state if (CurrentStateFunc() != &TThis::PendingActivation) { PassAwayTimestamp = TInstant::Max(); } - } + } TInstant PassAwayTimestamp; bool PassAwayScheduled = false; @@ -194,7 +194,7 @@ namespace NActors { {}, nullptr, 0)); PassAwayScheduled = true; } - } + } void HandlePassAwayIfNeeded() { Y_VERIFY(PassAwayScheduled); @@ -203,92 +203,92 @@ namespace NActors { } } - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // PendingActivation - // - // In this state we are just waiting for some activities, which may include: - // * an external Session event - // * incoming handshake request - // - // Upon receiving such event, we put it to corresponding queue and initiate start up by calling IssueGetNodeRequest, - // which, as the name says, issued TEvGetNode to the nameservice and arms timer to handle timeout (which should not - // occur, but we want to be sure we don't hang on this), and then switches to PendingNodeInfo state. - - PROXY_STFUNC(PendingActivation, - RequestNodeInfo, // Session events - RequestNodeInfoForIncomingHandshake, // Incoming handshake requests - Ignore, // Handshake status - Ignore, // Disconnect request - Ignore, // Wakeup - Ignore // Node info + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // PendingActivation + // + // In this state we are just waiting for some activities, which may include: + // * an external Session event + // * incoming handshake request + // + // Upon receiving such event, we put it to corresponding queue and initiate start up by calling IssueGetNodeRequest, + // which, as the name says, issued TEvGetNode to the nameservice and arms timer to handle timeout (which should not + // occur, but we want to be sure we don't hang on this), and then switches to PendingNodeInfo state. + + PROXY_STFUNC(PendingActivation, + RequestNodeInfo, // Session events + RequestNodeInfoForIncomingHandshake, // Incoming handshake requests + Ignore, // Handshake status + Ignore, // Disconnect request + Ignore, // Wakeup + Ignore // Node info ) - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // PendingNodeInfo - // - // This state is entered when we asked nameserver to provide description for peer node we are working with. All - // external Session events and incoming handshake requests are enqueued into their respective queues, TEvNodeInfo - // is main event that triggers processing. On success, we try to initiate outgoing handshake if needed, or process - // incoming handshakes. On error, we enter HoldByError state. - // - // NOTE: handshake status events are also enqueued as the handshake actor may have generated failure event due to - // timeout or some other reason without waiting for acknowledge, and it must be processed correctly to prevent - // session hang - - PROXY_STFUNC(PendingNodeInfo, - EnqueueSessionEvent, // Session events - EnqueueIncomingHandshakeEvent, // Incoming handshake requests - EnqueueIncomingHandshakeEvent, // Handshake status - Disconnect, // Disconnect request - ConfigureTimeout, // Wakeup - Configure // Node info + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // PendingNodeInfo + // + // This state is entered when we asked nameserver to provide description for peer node we are working with. All + // external Session events and incoming handshake requests are enqueued into their respective queues, TEvNodeInfo + // is main event that triggers processing. On success, we try to initiate outgoing handshake if needed, or process + // incoming handshakes. On error, we enter HoldByError state. + // + // NOTE: handshake status events are also enqueued as the handshake actor may have generated failure event due to + // timeout or some other reason without waiting for acknowledge, and it must be processed correctly to prevent + // session hang + + PROXY_STFUNC(PendingNodeInfo, + EnqueueSessionEvent, // Session events + EnqueueIncomingHandshakeEvent, // Incoming handshake requests + EnqueueIncomingHandshakeEvent, // Handshake status + Disconnect, // Disconnect request + ConfigureTimeout, // Wakeup + Configure // Node info ) - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // PendingConnection - // - // Here we have issued outgoing handshake or have accepted (or may be both) incoming handshake and we are waiting for - // the status of the handshake. When one if handshakes finishes, we use this status to establish connection (or to - // go to error state). When one handshake terminates with error while other is running, we will still wait for the - // second one to finish. - - PROXY_STFUNC(PendingConnection, - EnqueueSessionEvent, // Session events - IncomingHandshake, // Incoming handshake requests - HandleHandshakeStatus, // Handshake status - Disconnect, // Disconnect request - Ignore, // Wakeup - Ignore // Node info + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // PendingConnection + // + // Here we have issued outgoing handshake or have accepted (or may be both) incoming handshake and we are waiting for + // the status of the handshake. When one if handshakes finishes, we use this status to establish connection (or to + // go to error state). When one handshake terminates with error while other is running, we will still wait for the + // second one to finish. + + PROXY_STFUNC(PendingConnection, + EnqueueSessionEvent, // Session events + IncomingHandshake, // Incoming handshake requests + HandleHandshakeStatus, // Handshake status + Disconnect, // Disconnect request + Ignore, // Wakeup + Ignore // Node info ) - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // StateWork - // - // We have accepted session and process any incoming messages with the session. Incoming handshakes are accepted - // concurrently and applied when finished. - - PROXY_STFUNC(StateWork, - ForwardSessionEventToSession, // Session events - IncomingHandshake, // Incoming handshake requests - HandleHandshakeStatus, // Handshake status - Disconnect, // Disconnect request - Ignore, // Wakeup - Ignore // Node info + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // StateWork + // + // We have accepted session and process any incoming messages with the session. Incoming handshakes are accepted + // concurrently and applied when finished. + + PROXY_STFUNC(StateWork, + ForwardSessionEventToSession, // Session events + IncomingHandshake, // Incoming handshake requests + HandleHandshakeStatus, // Handshake status + Disconnect, // Disconnect request + Ignore, // Wakeup + Ignore // Node info ) - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // HoldByError - // - // When something bad happens with the connection, we sleep in this state. After wake up we go back to - // PendingActivation. - - PROXY_STFUNC(HoldByError, - DropSessionEvent, // Session events - RequestNodeInfoForIncomingHandshake, // Incoming handshake requests - Ignore, // Handshake status - Ignore, // Disconnect request - WakeupFromErrorState, // Wakeup - Ignore // Node info + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // HoldByError + // + // When something bad happens with the connection, we sleep in this state. After wake up we go back to + // PendingActivation. + + PROXY_STFUNC(HoldByError, + DropSessionEvent, // Session events + RequestNodeInfoForIncomingHandshake, // Incoming handshake requests + Ignore, // Handshake status + Ignore, // Disconnect request + WakeupFromErrorState, // Wakeup + Ignore // Node info ) #undef SESSION_EVENTS @@ -317,19 +317,19 @@ namespace NActors { void StartInitialHandshake(); void StartResumeHandshake(ui64 inputCounter); - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // Incoming handshake event queue processing - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // Incoming handshake event queue processing + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void EnqueueIncomingHandshakeEvent(STATEFN_SIG); void EnqueueIncomingHandshakeEvent(TEvHandshakeDone::TPtr& ev); void EnqueueIncomingHandshakeEvent(TEvHandshakeFail::TPtr& ev); - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // PendingNodeInfo - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // PendingNodeInfo + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - IEventBase* ConfigureTimeoutCookie; // pointer to the scheduled event used to match sent and received events + IEventBase* ConfigureTimeoutCookie; // pointer to the scheduled event used to match sent and received events void StartConfiguring(); void Configure(TEvInterconnect::TEvNodeInfo::TPtr& ev); @@ -343,7 +343,7 @@ namespace NActors { void WakeupFromErrorState(TEvents::TEvWakeup::TPtr& ev); void Disconnect(); - const ui32 PeerNodeId; + const ui32 PeerNodeId; IActor **DynamicPtr; void ValidateEvent(TAutoPtr<IEventHandle>& ev, const char* func) { @@ -362,15 +362,15 @@ namespace NActors { } // Common with helpers - // All proxy actors share the same information in the object - // read only + // All proxy actors share the same information in the object + // read only TInterconnectProxyCommon::TPtr const Common; const TActorId& GetNameserviceId() const { return Common->NameserviceId; - } + } - TString TechnicalPeerHostName; + TString TechnicalPeerHostName; std::shared_ptr<IInterconnectMetrics> Metrics; @@ -380,12 +380,12 @@ namespace NActors { void HandleSessionBufferSizeRequest(TEvSessionBufferSizeRequest::TPtr& ev); - bool CleanupEventQueueScheduled = false; + bool CleanupEventQueueScheduled = false; void ScheduleCleanupEventQueue(); void HandleCleanupEventQueue(); void CleanupEventQueue(); - // hold all events before connection is established + // hold all events before connection is established struct TPendingSessionEvent { TInstant Deadline; ui32 Size; @@ -402,12 +402,12 @@ namespace NActors { void ProcessPendingSessionEvents(); void DropSessionEvent(STATEFN_SIG); - TInterconnectSessionTCP* Session = nullptr; + TInterconnectSessionTCP* Session = nullptr; TActorId SessionID; - // virtual ids used during handshake to check if it is the connection - // for the same session or to find out the latest shandshake - // it's virtual because session actor apears after successfull handshake + // virtual ids used during handshake to check if it is the connection + // for the same session or to find out the latest shandshake + // it's virtual because session actor apears after successfull handshake TActorId SessionVirtualId; TActorId RemoteSessionVirtualId; @@ -416,27 +416,27 @@ namespace NActors { const ui64 localId = TlsActivationContext->ExecutorThread.ActorSystem->AllocateIDSpace(1); return NActors::TActorId(SelfId().NodeId(), 0, localId, 0); - } + } - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// TActorId IncomingHandshakeActor; - TInstant IncomingHandshakeActorFilledIn; - TInstant IncomingHandshakeActorReset; + TInstant IncomingHandshakeActorFilledIn; + TInstant IncomingHandshakeActorReset; TMaybe<ui64> LastSerialFromIncomingHandshake; - THolder<IEventBase> HeldHandshakeReply; + THolder<IEventBase> HeldHandshakeReply; void DropIncomingHandshake(bool poison = true) { ICPROXY_PROFILED; if (const TActorId& actorId = std::exchange(IncomingHandshakeActor, TActorId())) { LOG_DEBUG_IC("ICP111", "dropped incoming handshake: %s poison: %s", actorId.ToString().data(), - poison ? "true" : "false"); - if (poison) { + poison ? "true" : "false"); + if (poison) { Send(actorId, new TEvents::TEvPoisonPill); - } + } LastSerialFromIncomingHandshake.Clear(); - HeldHandshakeReply.Reset(); + HeldHandshakeReply.Reset(); IncomingHandshakeActorReset = TActivationContext::Now(); } } @@ -446,10 +446,10 @@ namespace NActors { if (const TActorId& actorId = std::exchange(OutgoingHandshakeActor, TActorId())) { LOG_DEBUG_IC("ICP112", "dropped outgoing handshake: %s poison: %s", actorId.ToString().data(), - poison ? "true" : "false"); - if (poison) { + poison ? "true" : "false"); + if (poison) { Send(actorId, new TEvents::TEvPoisonPill); - } + } OutgoingHandshakeActorReset = TActivationContext::Now(); } } @@ -464,38 +464,38 @@ namespace NActors { void PrepareNewSessionHandshake() { ICPROXY_PROFILED; - // drop existing session if we have one - if (Session) { - LOG_INFO_IC("ICP04", "terminating current session as we are negotiating a new one"); + // drop existing session if we have one + if (Session) { + LOG_INFO_IC("ICP04", "terminating current session as we are negotiating a new one"); IActor::InvokeOtherActor(*Session, &TInterconnectSessionTCP::Terminate, TDisconnectReason::NewSession()); - } + } - // ensure we have no current session - Y_VERIFY(!Session); + // ensure we have no current session + Y_VERIFY(!Session); - // switch to pending connection state -- we wait for handshakes, we want more handshakes! + // switch to pending connection state -- we wait for handshakes, we want more handshakes! SwitchToState(__LINE__, "PendingConnection", &TThis::PendingConnection); - } + } void IssueIncomingHandshakeReply(const TActorId& handshakeId, ui64 peerLocalId, - THolder<IEventBase> event); + THolder<IEventBase> event); - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// TActorId OutgoingHandshakeActor; - TInstant OutgoingHandshakeActorCreated; - TInstant OutgoingHandshakeActorReset; + TInstant OutgoingHandshakeActorCreated; + TInstant OutgoingHandshakeActorReset; - TInstant LastSessionDieTime; + TInstant LastSessionDieTime; void GenerateHttpInfo(NMon::TEvHttpInfo::TPtr& ev); void Handle(TEvQueryStats::TPtr& ev); - TDuration HoldByErrorWakeupDuration = TDuration::Zero(); - TEvents::TEvWakeup* HoldByErrorWakeupCookie; + TDuration HoldByErrorWakeupDuration = TDuration::Zero(); + TEvents::TEvWakeup* HoldByErrorWakeupCookie; - THolder<TProgramInfo> RemoteProgramInfo; + THolder<TProgramInfo> RemoteProgramInfo; NInterconnect::TSecureSocketContext::TPtr SecureContext; void Handle(TEvGetSecureSocket::TPtr ev) { @@ -503,11 +503,11 @@ namespace NActors { Send(ev->Sender, new TEvSecureSocket(std::move(socket))); } - TDeque<THolder<IEventHandle>> PendingIncomingHandshakeEvents; + TDeque<THolder<IEventHandle>> PendingIncomingHandshakeEvents; TDeque<std::tuple<TInstant, TString, TString, ui32>> ErrorStateLog; - - void UpdateErrorStateLog(TInstant now, TString kind, TString explanation) { + + void UpdateErrorStateLog(TInstant now, TString kind, TString explanation) { ICPROXY_PROFILED; if (ErrorStateLog) { @@ -521,9 +521,9 @@ namespace NActors { } ErrorStateLog.emplace_back(now, std::move(kind), std::move(explanation), 1); - if (ErrorStateLog.size() > 20) { - ErrorStateLog.pop_front(); - } + if (ErrorStateLog.size() > 20) { + ErrorStateLog.pop_front(); + } } void LogHandshakeFail(TEvHandshakeFail::TPtr& ev, bool inconclusive); @@ -532,6 +532,6 @@ namespace NActors { void HandleTerminate(); void PassAway() override; - }; + }; } diff --git a/library/cpp/actors/interconnect/interconnect_tcp_server.cpp b/library/cpp/actors/interconnect/interconnect_tcp_server.cpp index b95c994598..31bac08b29 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_server.cpp +++ b/library/cpp/actors/interconnect/interconnect_tcp_server.cpp @@ -8,15 +8,15 @@ namespace NActors { TInterconnectListenerTCP::TInterconnectListenerTCP(const TString& address, ui16 port, TInterconnectProxyCommon::TPtr common, const TMaybe<SOCKET>& socket) - : TActor(&TThis::Initial) + : TActor(&TThis::Initial) , TInterconnectLoggingBase(Sprintf("ICListener: %s", SelfId().ToString().data())) - , Address(address.c_str(), port) + , Address(address.c_str(), port) , Listener( socket ? new NInterconnect::TStreamSocket(*socket) : nullptr) , ExternalSocket(!!Listener) - , ProxyCommonCtx(std::move(common)) + , ProxyCommonCtx(std::move(common)) { if (ExternalSocket) { SetNonBlock(*Listener); @@ -24,13 +24,13 @@ namespace NActors { } TAutoPtr<IEventHandle> TInterconnectListenerTCP::AfterRegister(const TActorId& self, const TActorId& parentId) { - return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0); - } + return new IEventHandle(self, parentId, new TEvents::TEvBootstrap, 0); + } - void TInterconnectListenerTCP::Die(const TActorContext& ctx) { + void TInterconnectListenerTCP::Die(const TActorContext& ctx) { LOG_DEBUG_IC("ICL08", "Dying"); - TActor::Die(ctx); - } + TActor::Die(ctx); + } int TInterconnectListenerTCP::Bind() { NInterconnect::TAddress addr = Address; @@ -57,7 +57,7 @@ namespace NActors { Listener = NInterconnect::TStreamSocket::Make(addr.GetFamily()); if (*Listener == -1) { return errno; - } + } SetNonBlock(*Listener); Listener->SetSendBufferSize(ProxyCommonCtx->Settings.GetSendBufferSize()); // TODO(alexvru): WTF? SetSockOpt(*Listener, SOL_SOCKET, SO_REUSEADDR, 1); @@ -67,7 +67,7 @@ namespace NActors { return e; } else { return 0; - } + } } void TInterconnectListenerTCP::Bootstrap(const TActorContext& ctx) { @@ -78,7 +78,7 @@ namespace NActors { Become(&TThis::Initial, TDuration::Seconds(1), new TEvents::TEvBootstrap); return; } - } + } if (const auto& callback = ProxyCommonCtx->InitWhiteboard) { callback(Address.GetPort(), TlsActivationContext->ExecutorThread.ActorSystem); } @@ -112,6 +112,6 @@ namespace NActors { } break; } - } + } } diff --git a/library/cpp/actors/interconnect/interconnect_tcp_server.h b/library/cpp/actors/interconnect/interconnect_tcp_server.h index fc71073c2d..49e9f43dd6 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_server.h +++ b/library/cpp/actors/interconnect/interconnect_tcp_server.h @@ -9,46 +9,46 @@ #include "events_local.h" namespace NActors { - class TInterconnectListenerTCP: public TActor<TInterconnectListenerTCP>, public TInterconnectLoggingBase { - public: - static constexpr EActivityType ActorActivityType() { + class TInterconnectListenerTCP: public TActor<TInterconnectListenerTCP>, public TInterconnectLoggingBase { + public: + static constexpr EActivityType ActorActivityType() { return INTERCONNECT_COMMON; - } + } TInterconnectListenerTCP(const TString& address, ui16 port, TInterconnectProxyCommon::TPtr common, const TMaybe<SOCKET>& socket = Nothing()); int Bind(); - private: - STFUNC(Initial) { - switch (ev->GetTypeRewrite()) { - CFunc(TEvents::TEvBootstrap::EventType, Bootstrap); - CFunc(TEvents::TEvPoisonPill::EventType, Die); - } + private: + STFUNC(Initial) { + switch (ev->GetTypeRewrite()) { + CFunc(TEvents::TEvBootstrap::EventType, Bootstrap); + CFunc(TEvents::TEvPoisonPill::EventType, Die); + } } - STFUNC(Listen) { - switch (ev->GetTypeRewrite()) { - CFunc(TEvents::TEvPoisonPill::EventType, Die); + STFUNC(Listen) { + switch (ev->GetTypeRewrite()) { + CFunc(TEvents::TEvPoisonPill::EventType, Die); HFunc(TEvPollerRegisterResult, Handle); CFunc(TEvPollerReady::EventType, Process); - } + } } TAutoPtr<IEventHandle> AfterRegister(const TActorId& self, const TActorId& parentId) override; - void Die(const TActorContext& ctx) override; + void Die(const TActorContext& ctx) override; - void Bootstrap(const TActorContext& ctx); + void Bootstrap(const TActorContext& ctx); void Handle(TEvPollerRegisterResult::TPtr ev, const TActorContext& ctx); void Process(const TActorContext& ctx); - const NInterconnect::TAddress Address; - TIntrusivePtr<NInterconnect::TStreamSocket> Listener; + const NInterconnect::TAddress Address; + TIntrusivePtr<NInterconnect::TStreamSocket> Listener; const bool ExternalSocket; TPollerToken::TPtr PollerToken; TInterconnectProxyCommon::TPtr const ProxyCommonCtx; - }; + }; static inline TActorId MakeInterconnectListenerActorId(bool dynamic) { char x[12] = {'I', 'C', 'L', 'i', 's', 't', 'e', 'n', 'e', 'r', '/', dynamic ? 'D' : 'S'}; diff --git a/library/cpp/actors/interconnect/interconnect_tcp_session.cpp b/library/cpp/actors/interconnect/interconnect_tcp_session.cpp index 2ded7f9f53..0e18a20072 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_session.cpp +++ b/library/cpp/actors/interconnect/interconnect_tcp_session.cpp @@ -10,14 +10,14 @@ #include <library/cpp/monlib/service/pages/templates.h> namespace NActors { - LWTRACE_USING(ACTORLIB_PROVIDER); + LWTRACE_USING(ACTORLIB_PROVIDER); - DECLARE_WILSON_EVENT(OutputQueuePush, (ui32, QueueSizeInEvents), (ui64, QueueSizeInBytes)); + DECLARE_WILSON_EVENT(OutputQueuePush, (ui32, QueueSizeInEvents), (ui64, QueueSizeInBytes)); template<typename T> T Coalesce(T&& x) { return x; - } + } template<typename T, typename T2, typename... TRest> typename std::common_type<T, T2, TRest...>::type Coalesce(T&& first, T2&& mid, TRest&&... rest) { @@ -26,22 +26,22 @@ namespace NActors { } else { return Coalesce(std::forward<T2>(mid), std::forward<TRest>(rest)...); } - } + } TInterconnectSessionTCP::TInterconnectSessionTCP(TInterconnectProxyTCP* const proxy, TSessionParams params) - : TActor(&TInterconnectSessionTCP::StateFunc) + : TActor(&TInterconnectSessionTCP::StateFunc) , Created(TInstant::Now()) - , Proxy(proxy) + , Proxy(proxy) , CloseOnIdleWatchdog(GetCloseOnIdleTimeout(), std::bind(&TThis::OnCloseOnIdleTimerHit, this)) , LostConnectionWatchdog(GetLostConnectionTimeout(), std::bind(&TThis::OnLostConnectionTimerHit, this)) , Params(std::move(params)) - , TotalOutputQueueSize(0) - , OutputStuckFlag(false) - , OutputQueueUtilization(16) - , OutputCounter(0ULL) - { + , TotalOutputQueueSize(0) + , OutputStuckFlag(false) + , OutputQueueUtilization(16) + , OutputCounter(0ULL) + { Proxy->Metrics->SetConnected(0); - ReceiveContext.Reset(new TReceiveContext); + ReceiveContext.Reset(new TReceiveContext); } TInterconnectSessionTCP::~TInterconnectSessionTCP() { @@ -62,7 +62,7 @@ namespace NActors { LOG_INFO(*TlsActivationContext, NActorsServices::INTERCONNECT_STATUS, "[%u] session created", Proxy->PeerNodeId); SetPrefix(Sprintf("Session %s [node %" PRIu32 "]", SelfId().ToString().data(), Proxy->PeerNodeId)); SendUpdateToWhiteboard(); - } + } void TInterconnectSessionTCP::CloseInputSession() { Send(ReceiverId, new TEvInterconnect::TEvCloseInputSession); @@ -84,7 +84,7 @@ namespace NActors { for (const auto& kv : Subscribers) { Send(kv.first, new TEvInterconnect::TEvNodeDisconnected(Proxy->PeerNodeId), 0, kv.second); - } + } Proxy->Metrics->SubSubscribersCount(Subscribers.size()); Subscribers.clear(); @@ -108,7 +108,7 @@ namespace NActors { } TActor::PassAway(); - } + } void TInterconnectSessionTCP::PassAway() { Y_FAIL("TInterconnectSessionTCP::PassAway() can't be called directly"); @@ -118,13 +118,13 @@ namespace NActors { Proxy->ValidateEvent(ev, "Forward"); LOG_DEBUG_IC_SESSION("ICS02", "send event from: %s to: %s", ev->Sender.ToString().data(), ev->Recipient.ToString().data()); - ++MessagesGot; + ++MessagesGot; - if (ev->Flags & IEventHandle::FlagSubscribeOnSession) { + if (ev->Flags & IEventHandle::FlagSubscribeOnSession) { Subscribe(ev); - } + } - ui16 evChannel = ev->GetChannel(); + ui16 evChannel = ev->GetChannel(); auto& oChannel = ChannelScheduler->GetOutputChannel(evChannel); const bool wasWorking = oChannel.IsWorking(); @@ -136,31 +136,31 @@ namespace NActors { if (!wasWorking) { // this channel has returned to work -- it was empty and this we have just put first event in the queue ChannelScheduler->AddToHeap(oChannel, EqualizeCounter); - } + } - SetOutputStuckFlag(true); + SetOutputStuckFlag(true); ++NumEventsInReadyChannels; LWTRACK(EnqueueEvent, event->Orbit, Proxy->PeerNodeId, NumEventsInReadyChannels, GetWriteBlockedTotal(), evChannel, oChannel.GetQueueSize(), oChannel.GetBufferedAmountOfData()); WILSON_TRACE(*TlsActivationContext, &ev->TraceId, OutputQueuePush, - QueueSizeInEvents = oChannel.GetQueueSize(), + QueueSizeInEvents = oChannel.GetQueueSize(), QueueSizeInBytes = oChannel.GetBufferedAmountOfData()); - // check for overloaded queues + // check for overloaded queues ui64 sendBufferDieLimit = Proxy->Common->Settings.SendBufferDieLimitInMB * ui64(1 << 20); - if (sendBufferDieLimit != 0 && TotalOutputQueueSize > sendBufferDieLimit) { + if (sendBufferDieLimit != 0 && TotalOutputQueueSize > sendBufferDieLimit) { LOG_ERROR_IC_SESSION("ICS03", "socket: %" PRIi64 " output queue is overloaded, actual %" PRIu64 " bytes, limit is %" PRIu64, - Socket ? i64(*Socket) : -1, TotalOutputQueueSize, sendBufferDieLimit); + Socket ? i64(*Socket) : -1, TotalOutputQueueSize, sendBufferDieLimit); return Terminate(TDisconnectReason::QueueOverload()); - } + } ui64 outputBuffersTotalSizeLimit = Proxy->Common->Settings.OutputBuffersTotalSizeLimitInMB * ui64(1 << 20); if (outputBuffersTotalSizeLimit != 0 && static_cast<ui64>(Proxy->Metrics->GetOutputBuffersTotalSize()) > outputBuffersTotalSizeLimit) { LOG_ERROR_IC_SESSION("ICS77", "Exceeded total limit on output buffers size"); if (AtomicTryLock(&Proxy->Common->StartedSessionKiller)) { CreateSessionKillingActor(Proxy->Common); - } - } + } + } if (RamInQueue && !RamInQueue->Batching) { // we have pending TEvRam, so GenerateTraffic will be called no matter what @@ -198,12 +198,12 @@ namespace NActors { void TInterconnectSessionTCP::Unsubscribe(STATEFN_SIG) { LOG_DEBUG_IC_SESSION("ICS05", "unsubscribe for session state for %s", ev->Sender.ToString().data()); Proxy->Metrics->SubSubscribersCount( Subscribers.erase(ev->Sender)); - } + } THolder<TEvHandshakeAck> TInterconnectSessionTCP::ProcessHandshakeRequest(TEvHandshakeAsk::TPtr& ev) { TEvHandshakeAsk *msg = ev->Get(); - // close existing input session, if any, and do nothing upon its destruction + // close existing input session, if any, and do nothing upon its destruction ReestablishConnection({}, false, TDisconnectReason::NewSession()); const ui64 lastInputSerial = ReceiveContext->LockLastProcessedPacketSerial(); @@ -211,21 +211,21 @@ namespace NActors { msg->Self.ToString().data(), msg->Peer.ToString().data(), msg->Counter, lastInputSerial); return MakeHolder<TEvHandshakeAck>(msg->Peer, lastInputSerial, Params); - } + } void TInterconnectSessionTCP::SetNewConnection(TEvHandshakeDone::TPtr& ev) { - if (ReceiverId) { - // upon destruction of input session actor invoke this callback again + if (ReceiverId) { + // upon destruction of input session actor invoke this callback again ReestablishConnection(std::move(ev), false, TDisconnectReason::NewSession()); - return; - } + return; + } LOG_INFO_IC_SESSION("ICS09", "handshake done sender: %s self: %s peer: %s socket: %" PRIi64, ev->Sender.ToString().data(), ev->Get()->Self.ToString().data(), ev->Get()->Peer.ToString().data(), i64(*ev->Get()->Socket)); NewConnectionSet = TActivationContext::Now(); - PacketsWrittenToSocket = 0; + PacketsWrittenToSocket = 0; SendBufferSize = ev->Get()->Socket->GetSendBufferSize(); Socket = std::move(ev->Get()->Socket); @@ -233,21 +233,21 @@ namespace NActors { // there may be a race const ui64 nextPacket = Max(LastConfirmed, ev->Get()->NextPacket); - // arm watchdogs + // arm watchdogs CloseOnIdleWatchdog.Arm(SelfId()); - // reset activity timestamps + // reset activity timestamps LastInputActivityTimestamp = LastPayloadActivityTimestamp = TActivationContext::Now(); LOG_INFO_IC_SESSION("ICS10", "traffic start"); - // create input session actor + // create input session actor auto actor = MakeHolder<TInputSessionTCP>(SelfId(), Socket, ReceiveContext, Proxy->Common, Proxy->Metrics, Proxy->PeerNodeId, nextPacket, GetDeadPeerTimeout(), Params); ReceiveContext->UnlockLastProcessedPacketSerial(); ReceiverId = Params.Encryption ? RegisterWithSameMailbox(actor.Release()) : Register(actor.Release(), TMailboxType::ReadAsFilled); - // register our socket in poller actor + // register our socket in poller actor LOG_DEBUG_IC_SESSION("ICS11", "registering socket in PollerActor"); const bool success = Send(MakePollerActorId(), new TEvPollerRegister(Socket, ReceiverId, SelfId())); Y_VERIFY(success); @@ -257,31 +257,31 @@ namespace NActors { Proxy->Metrics->SetConnected(1); LOG_INFO(*TlsActivationContext, NActorsServices::INTERCONNECT_STATUS, "[%u] connected", Proxy->PeerNodeId); - // arm pinger timer + // arm pinger timer ResetFlushLogic(); - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - // REINITIALIZE SEND QUEUE - // - // scan through send queue and leave only those packets who have data -- we will simply resend them; drop all other + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // REINITIALIZE SEND QUEUE + // + // scan through send queue and leave only those packets who have data -- we will simply resend them; drop all other // auxiliary packets; also reset packet metrics to zero to start sending from the beginning - // also reset SendQueuePos + // also reset SendQueuePos - // drop confirmed packets first as we do not need unwanted retransmissions - SendQueuePos = SendQueue.end(); + // drop confirmed packets first as we do not need unwanted retransmissions + SendQueuePos = SendQueue.end(); DropConfirmed(nextPacket); for (TSendQueue::iterator it = SendQueue.begin(); it != SendQueue.end(); ) { - const TSendQueue::iterator next = std::next(it); - if (it->IsEmpty()) { + const TSendQueue::iterator next = std::next(it); + if (it->IsEmpty()) { SendQueueCache.splice(SendQueueCache.begin(), SendQueue, it); - } else { - it->ResetBufs(); - } - it = next; - } + } else { + it->ResetBufs(); + } + it = next; + } TrimSendQueueCache(); - SendQueuePos = SendQueue.begin(); + SendQueuePos = SendQueue.begin(); TMaybe<ui64> s; for (auto it = SendQueuePos; it != SendQueue.end(); ++it) { @@ -295,13 +295,13 @@ namespace NActors { LOG_DEBUG_IC_SESSION("ICS06", "rewind SendQueue size# %zu LastConfirmed# %" PRIu64 " SendQueuePos.Serial# %" PRIu64 "\n", SendQueue.size(), LastConfirmed, serial); - BytesUnwritten = 0; - for (const auto& packet : SendQueue) { + BytesUnwritten = 0; + for (const auto& packet : SendQueue) { BytesUnwritten += (Params.UseModernFrame ? sizeof(TTcpPacketHeader_v2) : sizeof(TTcpPacketHeader_v1)) + packet.GetDataSize(); - } + } - SwitchStuckPeriod(); + SwitchStuckPeriod(); LastHandshakeDone = TActivationContext::Now(); @@ -310,45 +310,45 @@ namespace NActors { } void TInterconnectSessionTCP::Handle(TEvUpdateFromInputSession::TPtr& ev) { - if (ev->Sender == ReceiverId) { - TEvUpdateFromInputSession& msg = *ev->Get(); + if (ev->Sender == ReceiverId) { + TEvUpdateFromInputSession& msg = *ev->Get(); // update ping time Ping = msg.Ping; LWPROBE(UpdateFromInputSession, Proxy->PeerNodeId, Ping.MillisecondsFloat()); - bool needConfirm = false; + bool needConfirm = false; - // update activity timer for dead peer checker + // update activity timer for dead peer checker LastInputActivityTimestamp = TActivationContext::Now(); if (msg.NumDataBytes) { - UnconfirmedBytes += msg.NumDataBytes; + UnconfirmedBytes += msg.NumDataBytes; if (UnconfirmedBytes >= GetTotalInflightAmountOfData() / 4) { - needConfirm = true; - } else { + needConfirm = true; + } else { SetForcePacketTimestamp(Proxy->Common->Settings.ForceConfirmPeriod); - } + } - // reset payload watchdog that controls close-on-idle behaviour + // reset payload watchdog that controls close-on-idle behaviour LastPayloadActivityTimestamp = TActivationContext::Now(); CloseOnIdleWatchdog.Reset(); - } + } bool unblockedSomething = false; LWPROBE_IF_TOO_LONG(SlowICDropConfirmed, Proxy->PeerNodeId, ms) { unblockedSomething = DropConfirmed(msg.ConfirmedByInput); - } + } // generate more traffic if we have unblocked state now if (unblockedSomething) { LWPROBE(UnblockByDropConfirmed, Proxy->PeerNodeId, NHPTimer::GetSeconds(GetCycleCountFast() - ev->SendTime) * 1000.0); GenerateTraffic(); - } + } - // if we haven't generated any packets, then make a lone Flush packet without any data - if (needConfirm && Socket) { - ++ConfirmPacketsForcedBySize; + // if we haven't generated any packets, then make a lone Flush packet without any data + if (needConfirm && Socket) { + ++ConfirmPacketsForcedBySize; MakePacket(false); } @@ -376,7 +376,7 @@ namespace NActors { } } } - } + } void TInterconnectSessionTCP::HandleRam(TEvRam::TPtr& ev) { if (ev->Get() == RamInQueue) { @@ -451,57 +451,57 @@ namespace NActors { void TInterconnectSessionTCP::StartHandshake() { LOG_INFO_IC_SESSION("ICS15", "start handshake"); IActor::InvokeOtherActor(*Proxy, &TInterconnectProxyTCP::StartResumeHandshake, ReceiveContext->LockLastProcessedPacketSerial()); - } + } void TInterconnectSessionTCP::ReestablishConnectionWithHandshake(TDisconnectReason reason) { ReestablishConnection({}, true, std::move(reason)); - } + } void TInterconnectSessionTCP::ReestablishConnection(TEvHandshakeDone::TPtr&& ev, bool startHandshakeOnSessionClose, TDisconnectReason reason) { - if (Socket) { + if (Socket) { LOG_INFO_IC_SESSION("ICS13", "reestablish connection"); ShutdownSocket(std::move(reason)); // stop sending/receiving on socket PendingHandshakeDoneEvent = std::move(ev); StartHandshakeOnSessionClose = startHandshakeOnSessionClose; - if (!ReceiverId) { + if (!ReceiverId) { ReestablishConnectionExecute(); - } + } } } void TInterconnectSessionTCP::OnDisconnect(TEvSocketDisconnect::TPtr& ev) { - if (ev->Sender == ReceiverId) { - const bool wasConnected(Socket); + if (ev->Sender == ReceiverId) { + const bool wasConnected(Socket); LOG_INFO_IC_SESSION("ICS07", "socket disconnect %" PRIi64 " reason# %s", Socket ? i64(*Socket) : -1, ev->Get()->Reason.ToString().data()); ReceiverId = TActorId(); // reset receiver actor id as we have no more receiver yet - if (wasConnected) { - // we were sucessfully connected and did not expect failure, so it arrived from the input side; we should - // restart handshake process, closing our part of socket first + if (wasConnected) { + // we were sucessfully connected and did not expect failure, so it arrived from the input side; we should + // restart handshake process, closing our part of socket first ShutdownSocket(ev->Get()->Reason); StartHandshake(); - } else { + } else { ReestablishConnectionExecute(); - } + } } } void TInterconnectSessionTCP::ShutdownSocket(TDisconnectReason reason) { - if (Socket) { + if (Socket) { if (const TString& s = reason.ToString()) { Proxy->Metrics->IncDisconnectByReason(s); } LOG_INFO_IC_SESSION("ICS25", "shutdown socket, reason# %s", reason.ToString().data()); Proxy->UpdateErrorStateLog(TActivationContext::Now(), "close_socket", reason.ToString().data()); - Socket->Shutdown(SHUT_RDWR); - Socket.Reset(); + Socket->Shutdown(SHUT_RDWR); + Socket.Reset(); Proxy->Metrics->IncDisconnections(); - CloseOnIdleWatchdog.Disarm(); + CloseOnIdleWatchdog.Disarm(); LostConnectionWatchdog.Arm(SelfId()); Proxy->Metrics->SetConnected(0); LOG_INFO(*TlsActivationContext, NActorsServices::INTERCONNECT_STATUS, "[%u] disconnected", Proxy->PeerNodeId); - } + } } void TInterconnectSessionTCP::ReestablishConnectionExecute() { @@ -512,7 +512,7 @@ namespace NActors { StartHandshake(); } else if (ev) { SetNewConnection(ev); - } + } } void TInterconnectSessionTCP::Handle(TEvPollerReady::TPtr& ev) { @@ -527,7 +527,7 @@ namespace NActors { GenerateTraffic(); } else if (!ev->Cookie) { Proxy->Metrics->IncSpuriousWriteWakeups(); - } + } if (Params.Encryption && ReceiveContext->ReadPending && !ev->Cookie) { Send(ReceiverId, ev->Release().Release(), 0, 1); } @@ -548,10 +548,10 @@ namespace NActors { void TInterconnectSessionTCP::WriteData() { ui64 written = 0; - Y_VERIFY(Socket); // ensure that socket wasn't closed + Y_VERIFY(Socket); // ensure that socket wasn't closed LWPROBE_IF_TOO_LONG(SlowICWriteData, Proxy->PeerNodeId, ms) { - constexpr ui32 iovLimit = 256; + constexpr ui32 iovLimit = 256; #ifdef _linux_ ui32 maxElementsInIOV = Min<ui32>(iovLimit, sysconf(_SC_IOV_MAX)); #else @@ -561,8 +561,8 @@ namespace NActors { maxElementsInIOV = 1; } - // vector of write buffers with preallocated stack space - TStackVec<TConstIoVec, iovLimit> wbuffers; + // vector of write buffers with preallocated stack space + TStackVec<TConstIoVec, iovLimit> wbuffers; LOG_DEBUG_IC_SESSION("ICS30", "WriteData WriteBlockedByFullSendBuffer# %s SendQueue.size# %zu", ReceiveContext->WriteBlockedByFullSendBuffer ? "true" : "false", SendQueue.size()); @@ -574,14 +574,14 @@ namespace NActors { while (SendQueuePos != SendQueue.end() && !ReceiveContext->WriteBlockedByFullSendBuffer) { for (auto it = SendQueuePos; it != SendQueue.end() && wbuffers.size() < maxElementsInIOV; ++it) { - it->AppendToIoVector(wbuffers, maxElementsInIOV); + it->AppendToIoVector(wbuffers, maxElementsInIOV); } - const struct iovec* iovec = reinterpret_cast<const struct iovec*>(wbuffers.data()); - int iovcnt = wbuffers.size(); + const struct iovec* iovec = reinterpret_cast<const struct iovec*>(wbuffers.data()); + int iovcnt = wbuffers.size(); - Y_VERIFY(iovcnt > 0); - Y_VERIFY(iovec->iov_len > 0); + Y_VERIFY(iovcnt > 0); + Y_VERIFY(iovec->iov_len > 0); TString err; ssize_t r = 0; @@ -596,23 +596,23 @@ namespace NActors { LOG_DEBUG_IC_SESSION("ICS16", "written# %zd iovcnt# %d err# %s", r, iovcnt, err.data()); - wbuffers.clear(); + wbuffers.clear(); - if (r > 0) { - Y_VERIFY(static_cast<size_t>(r) <= BytesUnwritten); - BytesUnwritten -= r; + if (r > 0) { + Y_VERIFY(static_cast<size_t>(r) <= BytesUnwritten); + BytesUnwritten -= r; written += r; ui64 packets = 0; - // advance SendQueuePos to eat all processed items - for (size_t amount = r; amount && SendQueuePos->DropBufs(amount); ++SendQueuePos) { + // advance SendQueuePos to eat all processed items + for (size_t amount = r; amount && SendQueuePos->DropBufs(amount); ++SendQueuePos) { if (!SendQueuePos->IsEmpty()) { LastSentSerial = Max(LastSentSerial, SendQueuePos->GetSerial()); } - ++PacketsWrittenToSocket; + ++PacketsWrittenToSocket; ++packets; LWTRACK(PacketWrittenToSocket, SendQueuePos->Orbit, Proxy->PeerNodeId, PacketsWrittenToSocket, SendQueuePos->TriedWriting, SendQueuePos->GetDataSize(), BytesUnwritten, GetWriteBlockedTotal(), (SOCKET)*Socket); - } + } LWPROBE(WriteToSocket, Proxy->PeerNodeId, r, packets, PacketsWrittenToSocket, BytesUnwritten, GetWriteBlockedTotal(), (SOCKET)*Socket); } else if (-r != EAGAIN && -r != EWOULDBLOCK) { @@ -624,15 +624,15 @@ namespace NActors { Proxy->Metrics->AddTotalBytesWritten(written); } return ReestablishConnectionWithHandshake(r == 0 ? TDisconnectReason::EndOfStream() : TDisconnectReason::FromErrno(-r)); - } else { + } else { // we have to do some hack for secure socket -- mark the packet as 'tried writing' if (Params.Encryption) { Y_VERIFY(SendQueuePos != SendQueue.end()); SendQueuePos->MarkTriedWriting(); // do not try to replace buffer under SSL } - // we have received EAGAIN error code, this means that we can't issue more data until we have received - // TEvPollerReadyWrite event from poller; set up flag meaning this and wait for that event + // we have received EAGAIN error code, this means that we can't issue more data until we have received + // TEvPollerReadyWrite event from poller; set up flag meaning this and wait for that event Y_VERIFY(!ReceiveContext->WriteBlockedByFullSendBuffer); ReceiveContext->WriteBlockedByFullSendBuffer = true; WriteBlockedCycles = GetCycleCountFast(); @@ -646,7 +646,7 @@ namespace NActors { } else { PollerToken->Request(false, true); } - } + } } } } @@ -656,12 +656,12 @@ namespace NActors { } void TInterconnectSessionTCP::SetForcePacketTimestamp(TDuration period) { - if (period != TDuration::Max()) { + if (period != TDuration::Max()) { const TInstant when = TActivationContext::Now() + period; - if (when < ForcePacketTimestamp) { - ForcePacketTimestamp = when; + if (when < ForcePacketTimestamp) { + ForcePacketTimestamp = when; ScheduleFlush(); - } + } } } @@ -671,7 +671,7 @@ namespace NActors { FlushSchedule.push(ForcePacketTimestamp); MaxFlushSchedule = Max(MaxFlushSchedule, FlushSchedule.size()); ++FlushEventsScheduled; - } + } } void TInterconnectSessionTCP::HandleFlush() { @@ -692,12 +692,12 @@ namespace NActors { } void TInterconnectSessionTCP::ResetFlushLogic() { - ForcePacketTimestamp = TInstant::Max(); - UnconfirmedBytes = 0; + ForcePacketTimestamp = TInstant::Max(); + UnconfirmedBytes = 0; const TDuration ping = Proxy->Common->Settings.PingPeriod; if (ping != TDuration::Zero() && !NumEventsInReadyChannels) { SetForcePacketTimestamp(ping); - } + } } void TInterconnectSessionTCP::TrimSendQueueCache() { @@ -719,19 +719,19 @@ namespace NActors { } ui64 TInterconnectSessionTCP::MakePacket(bool data, TMaybe<ui64> pingMask) { - Y_VERIFY(Socket); - - TSendQueue::iterator packet; - if (SendQueueCache) { - // we have entries in cache, take one and move it to the end of SendQueue - packet = SendQueueCache.begin(); - SendQueue.splice(SendQueue.end(), SendQueueCache, packet); - packet->Reuse(); // reset packet to initial state - } else { - // we have to allocate new packet, so just do it + Y_VERIFY(Socket); + + TSendQueue::iterator packet; + if (SendQueueCache) { + // we have entries in cache, take one and move it to the end of SendQueue + packet = SendQueueCache.begin(); + SendQueue.splice(SendQueue.end(), SendQueueCache, packet); + packet->Reuse(); // reset packet to initial state + } else { + // we have to allocate new packet, so just do it LWPROBE_IF_TOO_LONG(SlowICAllocPacketBuffer, Proxy->PeerNodeId, ms) { packet = SendQueue.emplace(SendQueue.end(), Params); - } + } } // update send queue position @@ -741,7 +741,7 @@ namespace NActors { ui64 serial = 0; - if (data) { + if (data) { // generate serial for this data packet serial = ++OutputCounter; @@ -749,21 +749,21 @@ namespace NActors { Y_VERIFY(NumEventsInReadyChannels); LWPROBE_IF_TOO_LONG(SlowICFillSendingBuffer, Proxy->PeerNodeId, ms) { FillSendingBuffer(*packet, serial); - } + } Y_VERIFY(!packet->IsEmpty()); InflightDataAmount += packet->GetDataSize(); Proxy->Metrics->AddInflightDataAmount(packet->GetDataSize()); if (InflightDataAmount > GetTotalInflightAmountOfData()) { Proxy->Metrics->IncInflyLimitReach(); - } + } - if (AtomicGet(ReceiveContext->ControlPacketId) == 0) { + if (AtomicGet(ReceiveContext->ControlPacketId) == 0) { AtomicSet(ReceiveContext->ControlPacketSendTimer, GetCycleCountFast()); - AtomicSet(ReceiveContext->ControlPacketId, OutputCounter); - } - - // update payload activity timer + AtomicSet(ReceiveContext->ControlPacketId, OutputCounter); + } + + // update payload activity timer LastPayloadActivityTimestamp = TActivationContext::Now(); } else if (pingMask) { serial = *pingMask; @@ -781,13 +781,13 @@ namespace NActors { SendQueue.splice(std::next(SendQueuePos), SendQueue, packet); } } - } + } const ui64 lastInputSerial = ReceiveContext->GetLastProcessedPacketSerial(); packet->SetMetadata(serial, lastInputSerial); - packet->Sign(); + packet->Sign(); - // count number of bytes pending for write + // count number of bytes pending for write ui64 packetSize = (Params.UseModernFrame ? sizeof(TTcpPacketHeader_v2) : sizeof(TTcpPacketHeader_v1)) + packet->GetDataSize(); BytesUnwritten += packetSize; @@ -795,15 +795,15 @@ namespace NActors { " InflightDataAmount# %" PRIu64 " BytesUnwritten# %" PRIu64, serial, lastInputSerial, packet->GetDataSize(), InflightDataAmount, BytesUnwritten); - // reset forced packet sending timestamp as we have confirmed all received data + // reset forced packet sending timestamp as we have confirmed all received data ResetFlushLogic(); - ++PacketsGenerated; + ++PacketsGenerated; LWTRACK(PacketGenerated, packet->Orbit, Proxy->PeerNodeId, BytesUnwritten, InflightDataAmount, PacketsGenerated, packetSize); - if (!data) { + if (!data) { WriteData(); - } + } return packetSize; } @@ -814,21 +814,21 @@ namespace NActors { Y_VERIFY(LastConfirmed <= confirm && confirm <= LastSentSerial && LastSentSerial <= OutputCounter, "%s confirm# %" PRIu64 " LastConfirmed# %" PRIu64 " OutputCounter# %" PRIu64 " LastSentSerial# %" PRIu64, LogPrefix.data(), confirm, LastConfirmed, OutputCounter, LastSentSerial); - LastConfirmed = confirm; + LastConfirmed = confirm; - ui64 droppedDataAmount = 0; - ui32 numDropped = 0; + ui64 droppedDataAmount = 0; + ui32 numDropped = 0; - // drop confirmed packets; this also includes any auxiliary packets as their serial is set to zero, effectively - // making Serial <= confirm true - TSendQueue::iterator it; + // drop confirmed packets; this also includes any auxiliary packets as their serial is set to zero, effectively + // making Serial <= confirm true + TSendQueue::iterator it; ui64 lastDroppedSerial = 0; for (it = SendQueue.begin(); it != SendQueuePos && it->Confirmed(confirm); ++it) { if (!it->IsEmpty()) { lastDroppedSerial = it->GetSerial(); } droppedDataAmount += it->GetDataSize(); - ++numDropped; + ++numDropped; } SendQueueCache.splice(SendQueueCache.begin(), SendQueue, SendQueue.begin(), it); TrimSendQueueCache(); @@ -840,7 +840,7 @@ namespace NActors { const ui64 limit = GetTotalInflightAmountOfData(); const bool unblockedSomething = current >= limit && current < limit + droppedDataAmount; - PacketsConfirmed += numDropped; + PacketsConfirmed += numDropped; InflightDataAmount -= droppedDataAmount; Proxy->Metrics->SubInflightDataAmount(droppedDataAmount); LWPROBE(DropConfirmed, Proxy->PeerNodeId, droppedDataAmount, InflightDataAmount); @@ -851,7 +851,7 @@ namespace NActors { Pool->Trim(); // send any unsent free requests return unblockedSomething; - } + } void TInterconnectSessionTCP::FillSendingBuffer(TTcpPacketOutTask& task, ui64 serial) { ui32 bytesGenerated = 0; @@ -914,42 +914,42 @@ namespace NActors { const ui32 utilization = Socket ? CalculateQueueUtilization() : 0; if (const auto& callback = Proxy->Common->UpdateWhiteboard) { - enum class EFlag { - GREEN, - YELLOW, - ORANGE, - RED, - }; - EFlag flagState = EFlag::RED; - - if (Socket) { - flagState = EFlag::GREEN; - - do { + enum class EFlag { + GREEN, + YELLOW, + ORANGE, + RED, + }; + EFlag flagState = EFlag::RED; + + if (Socket) { + flagState = EFlag::GREEN; + + do { auto lastInputDelay = TActivationContext::Now() - LastInputActivityTimestamp; - if (lastInputDelay * 4 >= GetDeadPeerTimeout() * 3) { - flagState = EFlag::ORANGE; - break; - } else if (lastInputDelay * 2 >= GetDeadPeerTimeout()) { - flagState = EFlag::YELLOW; - } - - // check utilization + if (lastInputDelay * 4 >= GetDeadPeerTimeout() * 3) { + flagState = EFlag::ORANGE; + break; + } else if (lastInputDelay * 2 >= GetDeadPeerTimeout()) { + flagState = EFlag::YELLOW; + } + + // check utilization if (utilization > 875000) { // 7/8 - flagState = EFlag::ORANGE; - break; + flagState = EFlag::ORANGE; + break; } else if (utilization > 500000) { // 1/2 - flagState = EFlag::YELLOW; - } - } while (false); - } + flagState = EFlag::YELLOW; + } + } while (false); + } callback(Proxy->Metrics->GetHumanFriendlyPeerHostName(), - connected, - flagState == EFlag::GREEN, - flagState == EFlag::YELLOW, - flagState == EFlag::ORANGE, - flagState == EFlag::RED, + connected, + flagState == EFlag::GREEN, + flagState == EFlag::YELLOW, + flagState == EFlag::ORANGE, + flagState == EFlag::RED, TlsActivationContext->ExecutorThread.ActorSystem); } @@ -958,34 +958,34 @@ namespace NActors { } } - void TInterconnectSessionTCP::SetOutputStuckFlag(bool state) { - if (OutputStuckFlag == state) - return; + void TInterconnectSessionTCP::SetOutputStuckFlag(bool state) { + if (OutputStuckFlag == state) + return; - if (OutputQueueUtilization.Size() == 0) - return; + if (OutputQueueUtilization.Size() == 0) + return; - auto& lastpair = OutputQueueUtilization.Last(); - if (state) + auto& lastpair = OutputQueueUtilization.Last(); + if (state) lastpair.first -= GetCycleCountFast(); - else + else lastpair.first += GetCycleCountFast(); - OutputStuckFlag = state; - } + OutputStuckFlag = state; + } - void TInterconnectSessionTCP::SwitchStuckPeriod() { + void TInterconnectSessionTCP::SwitchStuckPeriod() { auto now = GetCycleCountFast(); - if (OutputQueueUtilization.Size() != 0) { - auto& lastpair = OutputQueueUtilization.Last(); - lastpair.second = now - lastpair.second; - if (OutputStuckFlag) - lastpair.first += now; - } - - OutputQueueUtilization.Push(std::pair<ui64, ui64>(0, now)); + if (OutputQueueUtilization.Size() != 0) { + auto& lastpair = OutputQueueUtilization.Last(); + lastpair.second = now - lastpair.second; + if (OutputStuckFlag) + lastpair.first += now; + } + + OutputQueueUtilization.Push(std::pair<ui64, ui64>(0, now)); if (OutputStuckFlag) - OutputQueueUtilization.Last().first -= now; + OutputQueueUtilization.Last().first -= now; } TDuration TInterconnectSessionTCP::GetDeadPeerTimeout() const { @@ -1029,26 +1029,26 @@ namespace NActors { } void TInterconnectSessionTCP::GenerateHttpInfo(TStringStream& str) { - HTML(str) { - DIV_CLASS("panel panel-info") { - DIV_CLASS("panel-heading") { - str << "Session"; - } - DIV_CLASS("panel-body") { - TABLE_CLASS("table") { - TABLEHEAD() { - TABLER() { - TABLEH() { - str << "Sensor"; - } - TABLEH() { - str << "Value"; - } - } + HTML(str) { + DIV_CLASS("panel panel-info") { + DIV_CLASS("panel-heading") { + str << "Session"; + } + DIV_CLASS("panel-body") { + TABLE_CLASS("table") { + TABLEHEAD() { + TABLER() { + TABLEH() { + str << "Sensor"; + } + TABLEH() { + str << "Value"; + } + } } - TABLEBODY() { - TABLER() { - TABLED() { + TABLEBODY() { + TABLER() { + TABLED() { str << "Encryption"; } TABLED() { @@ -1111,89 +1111,89 @@ namespace NActors { } TABLER() { TABLED() { - str << "This page generated at"; - } - TABLED() { + str << "This page generated at"; + } + TABLED() { str << TActivationContext::Now() << " / " << Now(); - } - } - TABLER() { - TABLED() { - str << "SelfID"; - } - TABLED() { + } + } + TABLER() { + TABLED() { + str << "SelfID"; + } + TABLED() { str << SelfId().ToString(); - } - } + } + } TABLER() { TABLED() { str << "Frame version/Checksum"; } TABLED() { str << (!Params.UseModernFrame ? "v1/crc32c" : Params.Encryption ? "v2/none" : "v2/crc32c"); } } -#define MON_VAR(NAME) \ - TABLER() { \ - TABLED() { \ - str << #NAME; \ - } \ - TABLED() { \ - str << NAME; \ - } \ - } - - MON_VAR(Created) - MON_VAR(NewConnectionSet) - MON_VAR(ReceiverId) - MON_VAR(MessagesGot) - MON_VAR(MessagesWrittenToBuffer) - MON_VAR(PacketsGenerated) - MON_VAR(PacketsWrittenToSocket) - MON_VAR(PacketsConfirmed) - MON_VAR(AtomicGet(ReceiveContext->PacketsReadFromSocket)) - MON_VAR(ConfirmPacketsForcedBySize) - MON_VAR(ConfirmPacketsForcedByTimeout) - - TABLER() { - TABLED() { - str << "Virtual self ID"; - } - TABLED() { - str << Proxy->SessionVirtualId.ToString(); - } - } - TABLER() { - TABLED() { - str << "Virtual peer ID"; - } - TABLED() { - str << Proxy->RemoteSessionVirtualId.ToString(); - } - } - TABLER() { - TABLED() { - str << "Socket"; - } - TABLED() { - str << (Socket ? i64(*Socket) : -1); - } +#define MON_VAR(NAME) \ + TABLER() { \ + TABLED() { \ + str << #NAME; \ + } \ + TABLED() { \ + str << NAME; \ + } \ + } + + MON_VAR(Created) + MON_VAR(NewConnectionSet) + MON_VAR(ReceiverId) + MON_VAR(MessagesGot) + MON_VAR(MessagesWrittenToBuffer) + MON_VAR(PacketsGenerated) + MON_VAR(PacketsWrittenToSocket) + MON_VAR(PacketsConfirmed) + MON_VAR(AtomicGet(ReceiveContext->PacketsReadFromSocket)) + MON_VAR(ConfirmPacketsForcedBySize) + MON_VAR(ConfirmPacketsForcedByTimeout) + + TABLER() { + TABLED() { + str << "Virtual self ID"; + } + TABLED() { + str << Proxy->SessionVirtualId.ToString(); + } } + TABLER() { + TABLED() { + str << "Virtual peer ID"; + } + TABLED() { + str << Proxy->RemoteSessionVirtualId.ToString(); + } + } + TABLER() { + TABLED() { + str << "Socket"; + } + TABLED() { + str << (Socket ? i64(*Socket) : -1); + } + } ui32 unsentQueueSize = Socket ? Socket->GetUnsentQueueSize() : 0; - MON_VAR(OutputStuckFlag) - MON_VAR(SendQueue.size()) - MON_VAR(SendQueueCache.size()) + MON_VAR(OutputStuckFlag) + MON_VAR(SendQueue.size()) + MON_VAR(SendQueueCache.size()) MON_VAR(NumEventsInReadyChannels) - MON_VAR(TotalOutputQueueSize) - MON_VAR(BytesUnwritten) + MON_VAR(TotalOutputQueueSize) + MON_VAR(BytesUnwritten) MON_VAR(InflightDataAmount) MON_VAR(unsentQueueSize) MON_VAR(SendBufferSize) - MON_VAR(LastInputActivityTimestamp) - MON_VAR(LastPayloadActivityTimestamp) - MON_VAR(LastHandshakeDone) - MON_VAR(OutputCounter) + MON_VAR(LastInputActivityTimestamp) + MON_VAR(LastPayloadActivityTimestamp) + MON_VAR(LastHandshakeDone) + MON_VAR(OutputCounter) MON_VAR(LastSentSerial) MON_VAR(ReceiveContext->GetLastProcessedPacketSerial()) - MON_VAR(LastConfirmed) + MON_VAR(LastConfirmed) MON_VAR(FlushSchedule.size()) MON_VAR(MaxFlushSchedule) MON_VAR(FlushEventsScheduled) @@ -1211,11 +1211,11 @@ namespace NActors { MON_VAR(GetPingRTT()) MON_VAR(clockSkew) - MON_VAR(GetDeadPeerTimeout()) + MON_VAR(GetDeadPeerTimeout()) MON_VAR(GetTotalInflightAmountOfData()) MON_VAR(GetCloseOnIdleTimeout()) MON_VAR(Subscribers.size()) - } + } } } } @@ -1224,5 +1224,5 @@ namespace NActors { void CreateSessionKillingActor(TInterconnectProxyCommon::TPtr common) { TlsActivationContext->ExecutorThread.ActorSystem->Register(new TInterconnectSessionKiller(common)); - } + } } diff --git a/library/cpp/actors/interconnect/interconnect_tcp_session.h b/library/cpp/actors/interconnect/interconnect_tcp_session.h index 7fc00dbcc5..1ecd829132 100644 --- a/library/cpp/actors/interconnect/interconnect_tcp_session.h +++ b/library/cpp/actors/interconnect/interconnect_tcp_session.h @@ -30,61 +30,61 @@ #include <unordered_map> namespace NActors { - class TSlowPathChecker { - using TTraceCallback = std::function<void(double)>; - TTraceCallback Callback; - const NHPTimer::STime Start; - - public: - TSlowPathChecker(TTraceCallback&& callback) - : Callback(std::move(callback)) + class TSlowPathChecker { + using TTraceCallback = std::function<void(double)>; + TTraceCallback Callback; + const NHPTimer::STime Start; + + public: + TSlowPathChecker(TTraceCallback&& callback) + : Callback(std::move(callback)) , Start(GetCycleCountFast()) - { - } + { + } - ~TSlowPathChecker() { + ~TSlowPathChecker() { const NHPTimer::STime end = GetCycleCountFast(); - const NHPTimer::STime elapsed = end - Start; - if (elapsed > 1000000) { - Callback(NHPTimer::GetSeconds(elapsed) * 1000); - } + const NHPTimer::STime elapsed = end - Start; + if (elapsed > 1000000) { + Callback(NHPTimer::GetSeconds(elapsed) * 1000); + } + } + + operator bool() const { + return false; } + }; - operator bool() const { - return false; - } - }; +#define LWPROBE_IF_TOO_LONG(...) \ + if (auto __x = TSlowPathChecker{[&](double ms) { LWPROBE(__VA_ARGS__); }}) \ + ; \ + else -#define LWPROBE_IF_TOO_LONG(...) \ - if (auto __x = TSlowPathChecker{[&](double ms) { LWPROBE(__VA_ARGS__); }}) \ - ; \ - else - - class TTimeLimit { - public: - TTimeLimit(ui64 limitInCycles) + class TTimeLimit { + public: + TTimeLimit(ui64 limitInCycles) : UpperLimit(limitInCycles == 0 ? 0 : GetCycleCountFast() + limitInCycles) - { - } + { + } - TTimeLimit(ui64 startTS, ui64 limitInCycles) - : UpperLimit(limitInCycles == 0 ? 0 : startTS + limitInCycles) - { - } + TTimeLimit(ui64 startTS, ui64 limitInCycles) + : UpperLimit(limitInCycles == 0 ? 0 : startTS + limitInCycles) + { + } - bool CheckExceeded() { + bool CheckExceeded() { return UpperLimit != 0 && GetCycleCountFast() > UpperLimit; - } + } - const ui64 UpperLimit; - }; + const ui64 UpperLimit; + }; - static constexpr TDuration DEFAULT_DEADPEER_TIMEOUT = TDuration::Seconds(10); + static constexpr TDuration DEFAULT_DEADPEER_TIMEOUT = TDuration::Seconds(10); static constexpr TDuration DEFAULT_LOST_CONNECTION_TIMEOUT = TDuration::Seconds(10); - static constexpr ui32 DEFAULT_MAX_INFLIGHT_DATA = 10240 * 1024; - static constexpr ui32 DEFAULT_TOTAL_INFLIGHT_DATA = 4 * 10240 * 1024; + static constexpr ui32 DEFAULT_MAX_INFLIGHT_DATA = 10240 * 1024; + static constexpr ui32 DEFAULT_TOTAL_INFLIGHT_DATA = 4 * 10240 * 1024; - class TInterconnectProxyTCP; + class TInterconnectProxyTCP; enum class EUpdateState : ui8 { NONE, // no updates generated by input session yet @@ -93,22 +93,22 @@ namespace NActors { CONFIRMING, // confirmation inflight }; - struct TReceiveContext: public TAtomicRefCount<TReceiveContext> { - /* All invokations to these fields should be thread-safe */ + struct TReceiveContext: public TAtomicRefCount<TReceiveContext> { + /* All invokations to these fields should be thread-safe */ - ui64 ControlPacketSendTimer = 0; - ui64 ControlPacketId = 0; + ui64 ControlPacketSendTimer = 0; + ui64 ControlPacketId = 0; - // number of packets received by input session - TAtomic PacketsReadFromSocket = 0; - TAtomic DataPacketsReadFromSocket = 0; + // number of packets received by input session + TAtomic PacketsReadFromSocket = 0; + TAtomic DataPacketsReadFromSocket = 0; // last processed packet by input session std::atomic_uint64_t LastProcessedPacketSerial = 0; static constexpr uint64_t LastProcessedPacketSerialLockBit = uint64_t(1) << 63; - // for hardened checks - TAtomic NumInputSessions = 0; + // for hardened checks + TAtomic NumInputSessions = 0; NHPTimer::STime StartTime; @@ -160,9 +160,9 @@ namespace NActors { ui64 GetLastProcessedPacketSerial() { return LastProcessedPacketSerial.load() & ~LastProcessedPacketSerialLockBit; } - }; + }; - class TInputSessionTCP + class TInputSessionTCP : public TActorBootstrapped<TInputSessionTCP> , public TInterconnectLoggingBase { @@ -174,14 +174,14 @@ namespace NActors { struct TEvCheckDeadPeer : TEventLocal<TEvCheckDeadPeer, EvCheckDeadPeer> {}; struct TEvResumeReceiveData : TEventLocal<TEvResumeReceiveData, EvResumeReceiveData> {}; - public: - static constexpr EActivityType ActorActivityType() { + public: + static constexpr EActivityType ActorActivityType() { return INTERCONNECT_SESSION_TCP; - } + } TInputSessionTCP(const TActorId& sessionId, - TIntrusivePtr<NInterconnect::TStreamSocket> socket, - TIntrusivePtr<TReceiveContext> context, + TIntrusivePtr<NInterconnect::TStreamSocket> socket, + TIntrusivePtr<TReceiveContext> context, TInterconnectProxyCommon::TPtr common, std::shared_ptr<IInterconnectMetrics> metrics, ui32 nodeId, @@ -189,8 +189,8 @@ namespace NActors { TDuration deadPeerTimeout, TSessionParams params); - private: - friend class TActorBootstrapped<TInputSessionTCP>; + private: + friend class TActorBootstrapped<TInputSessionTCP>; void Bootstrap(); @@ -204,13 +204,13 @@ namespace NActors { cFunc(TEvConfirmUpdate::EventType, HandleConfirmUpdate) ) - private: + private: TRope IncomingData; const TActorId SessionId; - TIntrusivePtr<NInterconnect::TStreamSocket> Socket; + TIntrusivePtr<NInterconnect::TStreamSocket> Socket; TPollerToken::TPtr PollerToken; - TIntrusivePtr<TReceiveContext> Context; + TIntrusivePtr<TReceiveContext> Context; TInterconnectProxyCommon::TPtr Common; const ui32 NodeId; const TSessionParams Params; @@ -235,11 +235,11 @@ namespace NActors { THolder<TEvUpdateFromInputSession> UpdateFromInputSession; - ui64 ConfirmedByInput; + ui64 ConfirmedByInput; std::shared_ptr<IInterconnectMetrics> Metrics; - bool CloseInputSessionRequested = false; + bool CloseInputSessionRequested = false; void CloseInputSession(); @@ -258,12 +258,12 @@ namespace NActors { TDeque<TIntrusivePtr<TRopeAlignedBuffer>> Buffers; - static constexpr size_t NumPreallocatedBuffers = 16; - void PreallocateBuffers(); + static constexpr size_t NumPreallocatedBuffers = 16; + void PreallocateBuffers(); - inline ui64 GetMaxCyclesPerEvent() const { + inline ui64 GetMaxCyclesPerEvent() const { return DurationToCycles(TDuration::MicroSeconds(500)); - } + } const TDuration DeadPeerTimeout; TInstant LastReceiveTimestamp; @@ -278,21 +278,21 @@ namespace NActors { void HandlePingResponse(TDuration passed); void HandleClock(TInstant clock); - }; + }; - class TInterconnectSessionTCP + class TInterconnectSessionTCP : public TActor<TInterconnectSessionTCP> , public TInterconnectLoggingBase { - enum { + enum { EvCheckCloseOnIdle = EventSpaceBegin(TEvents::ES_PRIVATE), EvCheckLostConnection, EvRam, EvTerminate, EvFreeItems, - }; + }; - struct TEvCheckCloseOnIdle : TEventLocal<TEvCheckCloseOnIdle, EvCheckCloseOnIdle> {}; + struct TEvCheckCloseOnIdle : TEventLocal<TEvCheckCloseOnIdle, EvCheckCloseOnIdle> {}; struct TEvCheckLostConnection : TEventLocal<TEvCheckLostConnection, EvCheckLostConnection> {}; struct TEvRam : TEventLocal<TEvRam, EvRam> { @@ -308,18 +308,18 @@ namespace NActors { {} }; - const TInstant Created; - TInstant NewConnectionSet; - ui64 MessagesGot = 0; - ui64 MessagesWrittenToBuffer = 0; - ui64 PacketsGenerated = 0; - ui64 PacketsWrittenToSocket = 0; - ui64 PacketsConfirmed = 0; + const TInstant Created; + TInstant NewConnectionSet; + ui64 MessagesGot = 0; + ui64 MessagesWrittenToBuffer = 0; + ui64 PacketsGenerated = 0; + ui64 PacketsWrittenToSocket = 0; + ui64 PacketsConfirmed = 0; - public: - static constexpr EActivityType ActorActivityType() { - return INTERCONNECT_SESSION_TCP; - } + public: + static constexpr EActivityType ActorActivityType() { + return INTERCONNECT_SESSION_TCP; + } TInterconnectSessionTCP(TInterconnectProxyTCP* const proxy, TSessionParams params); ~TInterconnectSessionTCP(); @@ -339,8 +339,8 @@ namespace NActors { return ReceiveContext->ClockSkew_us; } - private: - friend class TInterconnectProxyTCP; + private: + friend class TInterconnectProxyTCP; void Handle(TEvTerminate::TPtr& ev); void HandlePoison(); @@ -400,7 +400,7 @@ namespace NActors { void ReestablishConnectionWithHandshake(TDisconnectReason reason); void ReestablishConnectionExecute(); - TInterconnectProxyTCP* const Proxy; + TInterconnectProxyTCP* const Proxy; // various connection settings access TDuration GetDeadPeerTimeout() const; @@ -418,11 +418,11 @@ namespace NActors { void IssuePingRequest(); void Handle(TEvProcessPingRequest::TPtr ev); - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - TInstant LastInputActivityTimestamp; - TInstant LastPayloadActivityTimestamp; - TWatchdogTimer<TEvCheckCloseOnIdle> CloseOnIdleWatchdog; + TInstant LastInputActivityTimestamp; + TInstant LastPayloadActivityTimestamp; + TWatchdogTimer<TEvCheckCloseOnIdle> CloseOnIdleWatchdog; TWatchdogTimer<TEvCheckLostConnection> LostConnectionWatchdog; void OnCloseOnIdleTimerHit() { @@ -435,26 +435,26 @@ namespace NActors { Terminate(TDisconnectReason::LostConnection()); } - //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////// const TSessionParams Params; TMaybe<TEventHolderPool> Pool; TMaybe<TChannelScheduler> ChannelScheduler; - ui64 TotalOutputQueueSize; - bool OutputStuckFlag; - TRecentWnd<std::pair<ui64, ui64>> OutputQueueUtilization; + ui64 TotalOutputQueueSize; + bool OutputStuckFlag; + TRecentWnd<std::pair<ui64, ui64>> OutputQueueUtilization; size_t NumEventsInReadyChannels = 0; - void SetOutputStuckFlag(bool state); - void SwitchStuckPeriod(); + void SetOutputStuckFlag(bool state); + void SwitchStuckPeriod(); - using TSendQueue = TList<TTcpPacketOutTask>; - TSendQueue SendQueue; - TSendQueue SendQueueCache; - TSendQueue::iterator SendQueuePos; + using TSendQueue = TList<TTcpPacketOutTask>; + TSendQueue SendQueue; + TSendQueue SendQueueCache; + TSendQueue::iterator SendQueuePos; ui64 WriteBlockedCycles = 0; // start of current block period TDuration WriteBlockedTotal; // total incremental duration that session has been blocked - ui64 BytesUnwritten = 0; + ui64 BytesUnwritten = 0; void TrimSendQueueCache(); @@ -467,21 +467,21 @@ namespace NActors { } } - ui64 OutputCounter; + ui64 OutputCounter; ui64 LastSentSerial = 0; - TInstant LastHandshakeDone; + TInstant LastHandshakeDone; - TIntrusivePtr<NInterconnect::TStreamSocket> Socket; + TIntrusivePtr<NInterconnect::TStreamSocket> Socket; TPollerToken::TPtr PollerToken; ui32 SendBufferSize; ui64 InflightDataAmount = 0; std::unordered_map<TActorId, ui64, TActorId::THash> Subscribers; - // time at which we want to send confirmation packet even if there was no outgoing data - ui64 UnconfirmedBytes = 0; - TInstant ForcePacketTimestamp = TInstant::Max(); + // time at which we want to send confirmation packet even if there was no outgoing data + ui64 UnconfirmedBytes = 0; + TInstant ForcePacketTimestamp = TInstant::Max(); TPriorityQueue<TInstant, TVector<TInstant>, std::greater<TInstant>> FlushSchedule; size_t MaxFlushSchedule = 0; ui64 FlushEventsScheduled = 0; @@ -494,71 +494,71 @@ namespace NActors { void GenerateHttpInfo(TStringStream& str); - TIntrusivePtr<TReceiveContext> ReceiveContext; + TIntrusivePtr<TReceiveContext> ReceiveContext; TActorId ReceiverId; TDuration Ping; - ui64 ConfirmPacketsForcedBySize = 0; - ui64 ConfirmPacketsForcedByTimeout = 0; + ui64 ConfirmPacketsForcedBySize = 0; + ui64 ConfirmPacketsForcedByTimeout = 0; - ui64 LastConfirmed = 0; + ui64 LastConfirmed = 0; TEvHandshakeDone::TPtr PendingHandshakeDoneEvent; bool StartHandshakeOnSessionClose = false; ui64 EqualizeCounter = 0; - }; + }; - class TInterconnectSessionKiller - : public TActorBootstrapped<TInterconnectSessionKiller> { - ui32 RepliesReceived = 0; - ui32 RepliesNumber = 0; + class TInterconnectSessionKiller + : public TActorBootstrapped<TInterconnectSessionKiller> { + ui32 RepliesReceived = 0; + ui32 RepliesNumber = 0; TActorId LargestSession = TActorId(); - ui64 MaxBufferSize = 0; + ui64 MaxBufferSize = 0; TInterconnectProxyCommon::TPtr Common; - public: + public: static constexpr EActivityType ActorActivityType() { return INTERCONNECT_SESSION_KILLER; } TInterconnectSessionKiller(TInterconnectProxyCommon::TPtr common) - : Common(common) - { - } + : Common(common) + { + } void Bootstrap() { - auto sender = SelfId(); + auto sender = SelfId(); const auto eventFabric = [&sender](const TActorId& recp) -> IEventHandle* { - auto ev = new TEvSessionBufferSizeRequest(); - return new IEventHandle(recp, sender, ev, IEventHandle::FlagTrackDelivery); - }; + auto ev = new TEvSessionBufferSizeRequest(); + return new IEventHandle(recp, sender, ev, IEventHandle::FlagTrackDelivery); + }; RepliesNumber = TlsActivationContext->ExecutorThread.ActorSystem->BroadcastToProxies(eventFabric); - Become(&TInterconnectSessionKiller::StateFunc); - } + Become(&TInterconnectSessionKiller::StateFunc); + } STRICT_STFUNC(StateFunc, hFunc(TEvSessionBufferSizeResponse, ProcessResponse) cFunc(TEvents::TEvUndelivered::EventType, ProcessUndelivered) ) - + void ProcessResponse(TEvSessionBufferSizeResponse::TPtr& ev) { - RepliesReceived++; - if (MaxBufferSize < ev->Get()->BufferSize) { - MaxBufferSize = ev->Get()->BufferSize; - LargestSession = ev->Get()->SessionID; - } - if (RepliesReceived == RepliesNumber) { + RepliesReceived++; + if (MaxBufferSize < ev->Get()->BufferSize) { + MaxBufferSize = ev->Get()->BufferSize; + LargestSession = ev->Get()->SessionID; + } + if (RepliesReceived == RepliesNumber) { Send(LargestSession, new TEvents::TEvPoisonPill); - AtomicUnlock(&Common->StartedSessionKiller); + AtomicUnlock(&Common->StartedSessionKiller); PassAway(); - } + } } - + void ProcessUndelivered() { - RepliesReceived++; + RepliesReceived++; } - }; + }; void CreateSessionKillingActor(TInterconnectProxyCommon::TPtr common); diff --git a/library/cpp/actors/interconnect/load.cpp b/library/cpp/actors/interconnect/load.cpp index 2a8443da71..f166ca0a99 100644 --- a/library/cpp/actors/interconnect/load.cpp +++ b/library/cpp/actors/interconnect/load.cpp @@ -34,9 +34,9 @@ namespace NInterconnect { CFunc(TEvents::TSystem::PoisonPill, Die); ) - void Handle(TEvLoadMessage::TPtr& ev, const TActorContext& ctx) { + void Handle(TEvLoadMessage::TPtr& ev, const TActorContext& ctx) { ui64 bytes = ev->Get()->CalculateSerializedSizeCached(); - auto& record = ev->Get()->Record; + auto& record = ev->Get()->Record; auto *hops = record.MutableHops(); while (!hops->empty() && !hops->begin()->HasNextHop()) { record.ClearPayload(); @@ -81,7 +81,7 @@ namespace NInterconnect { CFunc(TEvents::TSystem::PoisonPill, Die); ) - void Handle(TEvLoadMessage::TPtr& ev, const TActorContext& ctx) { + void Handle(TEvLoadMessage::TPtr& ev, const TActorContext& ctx) { ctx.ExecutorThread.ActorSystem->Send(ev->Forward(Slaves[SlaveIndex])); if (++SlaveIndex == Slaves.size()) { SlaveIndex = 0; @@ -107,7 +107,7 @@ namespace NInterconnect { TLoadResponderMasterActor() {} - void Bootstrap(const TActorContext& ctx) { + void Bootstrap(const TActorContext& ctx) { Become(&TLoadResponderMasterActor::StateFunc); while (Slaves.size() < 10) { Slaves.push_back(ctx.Register(new TLoadResponderActor(Traffic))); @@ -118,7 +118,7 @@ namespace NInterconnect { std::shared_ptr<std::atomic_uint64_t> Traffic = std::make_shared<std::atomic_uint64_t>(); }; - IActor* CreateLoadResponderActor() { + IActor* CreateLoadResponderActor() { return new TLoadResponderMasterActor(); } @@ -127,17 +127,17 @@ namespace NInterconnect { return TActorId(nodeId, TStringBuf(x, 12)); } - class TLoadActor: public TActorBootstrapped<TLoadActor> { + class TLoadActor: public TActorBootstrapped<TLoadActor> { struct TEvGenerateMessages : TEventLocal<TEvGenerateMessages, EvGenerateMessages> {}; struct TEvPublishResults : TEventLocal<TEvPublishResults, EvPublishResults> {}; struct TMessageInfo { TInstant SendTimestamp; - TMessageInfo(const TInstant& sendTimestamp) + TMessageInfo(const TInstant& sendTimestamp) : SendTimestamp(sendTimestamp) - { - } + { + } }; const TLoadParams Params; @@ -154,14 +154,14 @@ namespace NInterconnect { return IActor::INTERCONNECT_LOAD_ACTOR; } - TLoadActor(const TLoadParams& params) + TLoadActor(const TLoadParams& params) : Params(params) {} void Bootstrap(const TActorContext& ctx) { Become(&TLoadActor::QueryTrafficCounter); ctx.Send(MakeLoadResponderActorId(SelfId().NodeId()), new TEvQueryTrafficCounter); - } + } void Handle(TEvTrafficCounter::TPtr ev, const TActorContext& ctx) { Traffic = std::move(ev->Get()->Traffic); @@ -185,7 +185,7 @@ namespace NInterconnect { SchedulePublishResults(ctx); } - void GenerateMessages(const TActorContext& ctx) { + void GenerateMessages(const TActorContext& ctx) { while (InFly.size() < Params.InFlyMax && ctx.Now() >= NextMessageTimestamp) { // generate payload const ui32 size = Params.SizeMin + RandomNumber(Params.SizeMax - Params.SizeMin + 1); @@ -232,8 +232,8 @@ namespace NInterconnect { } } - void Handle(TEvLoadMessage::TPtr& ev, const TActorContext& ctx) { - const auto& record = ev->Get()->Record; + void Handle(TEvLoadMessage::TPtr& ev, const TActorContext& ctx) { + const auto& record = ev->Get()->Record; auto it = InFly.find(record.GetId()); if (it != InFly.end()) { // record message rtt @@ -294,18 +294,18 @@ namespace NInterconnect { TQueue<std::pair<TInstant, TString>> TimeoutQueue; - void PutTimeoutQueueItem(const TActorContext& ctx, TString id) { + void PutTimeoutQueueItem(const TActorContext& ctx, TString id) { TimeoutQueue.emplace(ctx.Now() + TDuration::Minutes(1), std::move(id)); if (TimeoutQueue.size() == 1) { ScheduleWakeup(ctx); } } - void ScheduleWakeup(const TActorContext& ctx) { + void ScheduleWakeup(const TActorContext& ctx) { ctx.Schedule(TimeoutQueue.front().first - ctx.Now(), new TEvents::TEvWakeup); } - void HandleWakeup(const TActorContext& ctx) { + void HandleWakeup(const TActorContext& ctx) { ui32 numDropped = 0; while (TimeoutQueue && TimeoutQueue.front().first <= ctx.Now()) { @@ -326,11 +326,11 @@ namespace NInterconnect { const TDuration ResultPublishPeriod = TDuration::Seconds(15); - void SchedulePublishResults(const TActorContext& ctx) { + void SchedulePublishResults(const TActorContext& ctx) { ctx.Schedule(ResultPublishPeriod, new TEvPublishResults); } - void PublishResults(const TActorContext& ctx, bool schedule = true) { + void PublishResults(const TActorContext& ctx, bool schedule = true) { const TInstant now = ctx.Now(); TStringStream msg; @@ -354,7 +354,7 @@ namespace NInterconnect { msg << "{window# " << duration << " samples# " << Histogram.size(); TVector<TDuration> v; v.reserve(Histogram.size()); - for (const auto& item : Histogram) { + for (const auto& item : Histogram) { v.push_back(item.second); } std::sort(v.begin(), v.end()); @@ -398,8 +398,8 @@ namespace NInterconnect { } }; - IActor* CreateLoadActor(const TLoadParams& params) { + IActor* CreateLoadActor(const TLoadParams& params) { return new TLoadActor(params); } -} +} diff --git a/library/cpp/actors/interconnect/load.h b/library/cpp/actors/interconnect/load.h index 0a01a0dc04..78c80c8e17 100644 --- a/library/cpp/actors/interconnect/load.h +++ b/library/cpp/actors/interconnect/load.h @@ -4,21 +4,21 @@ namespace NInterconnect { // load responder -- lives on every node as a service actor - NActors::IActor* CreateLoadResponderActor(); + NActors::IActor* CreateLoadResponderActor(); NActors::TActorId MakeLoadResponderActorId(ui32 node); // load actor -- generates load with specific parameters struct TLoadParams { TString Name; ui32 Channel; - TVector<ui32> NodeHops; // node ids for the message route - ui32 SizeMin, SizeMax; // min and max size for payloads - ui32 InFlyMax; // maximum number of in fly messages + TVector<ui32> NodeHops; // node ids for the message route + ui32 SizeMin, SizeMax; // min and max size for payloads + ui32 InFlyMax; // maximum number of in fly messages TDuration IntervalMin, IntervalMax; // min and max intervals between sending messages - bool SoftLoad; // is the load soft? - TDuration Duration; // test duration + bool SoftLoad; // is the load soft? + TDuration Duration; // test duration bool UseProtobufWithPayload; // store payload separately }; - NActors::IActor* CreateLoadActor(const TLoadParams& params); + NActors::IActor* CreateLoadActor(const TLoadParams& params); -} +} diff --git a/library/cpp/actors/interconnect/logging.h b/library/cpp/actors/interconnect/logging.h index c429d1cade..4d628ba859 100644 --- a/library/cpp/actors/interconnect/logging.h +++ b/library/cpp/actors/interconnect/logging.h @@ -18,7 +18,7 @@ #define LOG_LOG_IC(component, marker, priority, ...) \ do { \ LOG_LOG(::NActors::TActivationContext::AsActorContext(), (priority), (component), "%s " marker " %s", LogPrefix.data(), Sprintf(__VA_ARGS__).data()); \ - } while (false) + } while (false) #define LOG_LOG_NET(priority, NODE_ID, FMT, ...) \ do { \ @@ -58,8 +58,8 @@ namespace NActors { TInterconnectLoggingBase(const TString& prefix) : LogPrefix(prefix) - { - } + { + } void SetPrefix(TString logPrefix) const { logPrefix.swap(const_cast<TString&>(LogPrefix)); diff --git a/library/cpp/actors/interconnect/packet.h b/library/cpp/actors/interconnect/packet.h index 4ba50a2b5f..e7cbb453f6 100644 --- a/library/cpp/actors/interconnect/packet.h +++ b/library/cpp/actors/interconnect/packet.h @@ -16,10 +16,10 @@ #define FORCE_EVENT_CHECKSUM 0 #endif -using NActors::IEventBase; -using NActors::IEventHandle; +using NActors::IEventBase; +using NActors::IEventHandle; using NActors::TActorId; -using NActors::TConstIoVec; +using NActors::TConstIoVec; using NActors::TEventSerializedData; Y_FORCE_INLINE ui32 Crc32cExtendMSanCompatible(ui32 checksum, const void *data, size_t len) { @@ -142,7 +142,7 @@ struct TEventHolder : TNonCopyable { namespace NActors { class TEventOutputChannel; -} +} struct TTcpPacketOutTask : TNonCopyable { const TSessionParams& Params; @@ -192,9 +192,9 @@ public: Orbit.Reset(); } - bool IsEmpty() const { + bool IsEmpty() const { return !DataSize; - } + } void SetMetadata(ui64 serial, ui64 confirm) { ApplyToHeader([&](auto& header) { @@ -261,7 +261,7 @@ public: bool DropBufs(size_t& amount) { while (BufferIndex != Bufs.size()) { - TConstIoVec& item = Bufs[BufferIndex]; + TConstIoVec& item = Bufs[BufferIndex]; // calculate number of bytes to the end in current buffer const size_t remain = item.Size - FirstBufferOffset; if (amount >= remain) { @@ -285,11 +285,11 @@ public: TriedWriting = false; } - template <typename TVectorType> - void AppendToIoVector(TVectorType& vector, size_t max) { + template <typename TVectorType> + void AppendToIoVector(TVectorType& vector, size_t max) { for (size_t k = BufferIndex, offset = FirstBufferOffset; k != Bufs.size() && vector.size() < max; ++k, offset = 0) { TConstIoVec v = Bufs[k]; - v.Data = static_cast<const char*>(v.Data) + offset; + v.Data = static_cast<const char*>(v.Data) + offset; v.Size -= offset; vector.push_back(v); } diff --git a/library/cpp/actors/interconnect/poller.h b/library/cpp/actors/interconnect/poller.h index ff7979369f..476a7a5e92 100644 --- a/library/cpp/actors/interconnect/poller.h +++ b/library/cpp/actors/interconnect/poller.h @@ -4,20 +4,20 @@ #include <library/cpp/actors/core/events.h> namespace NActors { - class TSharedDescriptor: public TThrRefBase { - public: - virtual int GetDescriptor() = 0; - }; + class TSharedDescriptor: public TThrRefBase { + public: + virtual int GetDescriptor() = 0; + }; - using TDelegate = std::function<void()>; - using TFDDelegate = std::function<TDelegate(const TIntrusivePtr<TSharedDescriptor>&)>; + using TDelegate = std::function<void()>; + using TFDDelegate = std::function<TDelegate(const TIntrusivePtr<TSharedDescriptor>&)>; - class IPoller: public TThrRefBase { - public: - virtual ~IPoller() = default; + class IPoller: public TThrRefBase { + public: + virtual ~IPoller() = default; - virtual void StartRead(const TIntrusivePtr<TSharedDescriptor>& s, TFDDelegate&& operation) = 0; - virtual void StartWrite(const TIntrusivePtr<TSharedDescriptor>& s, TFDDelegate&& operation) = 0; - }; + virtual void StartRead(const TIntrusivePtr<TSharedDescriptor>& s, TFDDelegate&& operation) = 0; + virtual void StartWrite(const TIntrusivePtr<TSharedDescriptor>& s, TFDDelegate&& operation) = 0; + }; } diff --git a/library/cpp/actors/interconnect/poller_actor.cpp b/library/cpp/actors/interconnect/poller_actor.cpp index e75cbcaef4..930d334821 100644 --- a/library/cpp/actors/interconnect/poller_actor.cpp +++ b/library/cpp/actors/interconnect/poller_actor.cpp @@ -36,7 +36,7 @@ namespace NActors { const TActorId WriteActorId; std::atomic_uint32_t Flags = 0; - TSocketRecord(TEvPollerRegister& ev) + TSocketRecord(TEvPollerRegister& ev) : Socket(std::move(ev.Socket)) , ReadActorId(ev.ReadActorId) , WriteActorId(ev.WriteActorId) @@ -287,8 +287,8 @@ namespace NActors { Impl->Request(read, write); } - IActor* CreatePollerActor() { + IActor* CreatePollerActor() { return new TPollerActor; } -} +} diff --git a/library/cpp/actors/interconnect/poller_actor.h b/library/cpp/actors/interconnect/poller_actor.h index f927b82089..899dbb73e1 100644 --- a/library/cpp/actors/interconnect/poller_actor.h +++ b/library/cpp/actors/interconnect/poller_actor.h @@ -53,11 +53,11 @@ namespace NActors { {} }; - IActor* CreatePollerActor(); + IActor* CreatePollerActor(); inline TActorId MakePollerActorId() { char x[12] = {'I', 'C', 'P', 'o', 'l', 'l', 'e', 'r', '\xDE', '\xAD', '\xBE', '\xEF'}; return TActorId(0, TStringBuf(std::begin(x), std::end(x))); } -} +} diff --git a/library/cpp/actors/interconnect/poller_tcp.cpp b/library/cpp/actors/interconnect/poller_tcp.cpp index 8267df31ea..456bca8b98 100644 --- a/library/cpp/actors/interconnect/poller_tcp.cpp +++ b/library/cpp/actors/interconnect/poller_tcp.cpp @@ -1,35 +1,35 @@ #include "poller_tcp.h" namespace NInterconnect { - TPollerThreads::TPollerThreads(size_t units, bool useSelect) - : Units(units) - { - Y_VERIFY_DEBUG(!Units.empty()); - for (auto& unit : Units) - unit = TPollerUnit::Make(useSelect); - } + TPollerThreads::TPollerThreads(size_t units, bool useSelect) + : Units(units) + { + Y_VERIFY_DEBUG(!Units.empty()); + for (auto& unit : Units) + unit = TPollerUnit::Make(useSelect); + } - TPollerThreads::~TPollerThreads() { - } + TPollerThreads::~TPollerThreads() { + } - void TPollerThreads::Start() { - for (const auto& unit : Units) - unit->Start(); - } + void TPollerThreads::Start() { + for (const auto& unit : Units) + unit->Start(); + } - void TPollerThreads::Stop() { - for (const auto& unit : Units) - unit->Stop(); - } + void TPollerThreads::Stop() { + for (const auto& unit : Units) + unit->Stop(); + } - void TPollerThreads::StartRead(const TIntrusivePtr<TSharedDescriptor>& s, TFDDelegate&& operation) { - auto& unit = Units[THash<SOCKET>()(s->GetDescriptor()) % Units.size()]; - unit->StartReadOperation(s, std::move(operation)); - } + void TPollerThreads::StartRead(const TIntrusivePtr<TSharedDescriptor>& s, TFDDelegate&& operation) { + auto& unit = Units[THash<SOCKET>()(s->GetDescriptor()) % Units.size()]; + unit->StartReadOperation(s, std::move(operation)); + } - void TPollerThreads::StartWrite(const TIntrusivePtr<TSharedDescriptor>& s, TFDDelegate&& operation) { - auto& unit = Units[THash<SOCKET>()(s->GetDescriptor()) % Units.size()]; - unit->StartWriteOperation(s, std::move(operation)); - } + void TPollerThreads::StartWrite(const TIntrusivePtr<TSharedDescriptor>& s, TFDDelegate&& operation) { + auto& unit = Units[THash<SOCKET>()(s->GetDescriptor()) % Units.size()]; + unit->StartWriteOperation(s, std::move(operation)); + } } diff --git a/library/cpp/actors/interconnect/poller_tcp.h b/library/cpp/actors/interconnect/poller_tcp.h index 310265eccd..2ae8631237 100644 --- a/library/cpp/actors/interconnect/poller_tcp.h +++ b/library/cpp/actors/interconnect/poller_tcp.h @@ -7,19 +7,19 @@ #include <util/generic/hash.h> namespace NInterconnect { - class TPollerThreads: public NActors::IPoller { - public: - TPollerThreads(size_t units = 1U, bool useSelect = false); - ~TPollerThreads(); + class TPollerThreads: public NActors::IPoller { + public: + TPollerThreads(size_t units = 1U, bool useSelect = false); + ~TPollerThreads(); - void Start(); - void Stop(); + void Start(); + void Stop(); - void StartRead(const TIntrusivePtr<TSharedDescriptor>& s, TFDDelegate&& operation) override; - void StartWrite(const TIntrusivePtr<TSharedDescriptor>& s, TFDDelegate&& operation) override; + void StartRead(const TIntrusivePtr<TSharedDescriptor>& s, TFDDelegate&& operation) override; + void StartWrite(const TIntrusivePtr<TSharedDescriptor>& s, TFDDelegate&& operation) override; - private: - TVector<TPollerUnit::TPtr> Units; - }; + private: + TVector<TPollerUnit::TPtr> Units; + }; } diff --git a/library/cpp/actors/interconnect/poller_tcp_unit.cpp b/library/cpp/actors/interconnect/poller_tcp_unit.cpp index 59e7dda810..36180353b6 100644 --- a/library/cpp/actors/interconnect/poller_tcp_unit.cpp +++ b/library/cpp/actors/interconnect/poller_tcp_unit.cpp @@ -1,7 +1,7 @@ #include "poller_tcp_unit.h" #if !defined(_win_) && !defined(_darwin_) -#include "poller_tcp_unit_epoll.h" +#include "poller_tcp_unit_epoll.h" #endif #include "poller_tcp_unit_select.h" @@ -11,116 +11,116 @@ #include <library/cpp/actors/util/intrinsics.h> #if defined _linux_ -#include <pthread.h> +#include <pthread.h> #endif namespace NInterconnect { - TPollerUnit::TPtr - TPollerUnit::Make(bool useSelect) { + TPollerUnit::TPtr + TPollerUnit::Make(bool useSelect) { #if defined(_win_) || defined(_darwin_) - Y_UNUSED(useSelect); - return TPtr(new TPollerUnitSelect); + Y_UNUSED(useSelect); + return TPtr(new TPollerUnitSelect); #else - return useSelect ? TPtr(new TPollerUnitSelect) : TPtr(new TPollerUnitEpoll); + return useSelect ? TPtr(new TPollerUnitSelect) : TPtr(new TPollerUnitEpoll); #endif - } - - TPollerUnit::TPollerUnit() - : StopFlag(true) - , ReadLoop(TThread::TParams(IdleThread<false>, this).SetName("network read")) - , WriteLoop(TThread::TParams(IdleThread<true>, this).SetName("network write")) - { - } - - TPollerUnit::~TPollerUnit() { - if (!AtomicLoad(&StopFlag)) - Stop(); - } - - void - TPollerUnit::Start() { - AtomicStore(&StopFlag, false); - ReadLoop.Start(); - WriteLoop.Start(); - } - - void - TPollerUnit::Stop() { - AtomicStore(&StopFlag, true); - ReadLoop.Join(); - WriteLoop.Join(); - } - - template <> - TPollerUnit::TSide& - TPollerUnit::GetSide<false>() { - return Read; - } - - template <> - TPollerUnit::TSide& - TPollerUnit::GetSide<true>() { - return Write; - } - - void - TPollerUnit::StartReadOperation( + } + + TPollerUnit::TPollerUnit() + : StopFlag(true) + , ReadLoop(TThread::TParams(IdleThread<false>, this).SetName("network read")) + , WriteLoop(TThread::TParams(IdleThread<true>, this).SetName("network write")) + { + } + + TPollerUnit::~TPollerUnit() { + if (!AtomicLoad(&StopFlag)) + Stop(); + } + + void + TPollerUnit::Start() { + AtomicStore(&StopFlag, false); + ReadLoop.Start(); + WriteLoop.Start(); + } + + void + TPollerUnit::Stop() { + AtomicStore(&StopFlag, true); + ReadLoop.Join(); + WriteLoop.Join(); + } + + template <> + TPollerUnit::TSide& + TPollerUnit::GetSide<false>() { + return Read; + } + + template <> + TPollerUnit::TSide& + TPollerUnit::GetSide<true>() { + return Write; + } + + void + TPollerUnit::StartReadOperation( const TIntrusivePtr<TSharedDescriptor>& stream, - TFDDelegate&& operation) { - Y_VERIFY_DEBUG(stream); - if (AtomicLoad(&StopFlag)) - return; - GetSide<false>().InputQueue.Push(TSide::TItem(stream, std::move(operation))); - } - - void - TPollerUnit::StartWriteOperation( + TFDDelegate&& operation) { + Y_VERIFY_DEBUG(stream); + if (AtomicLoad(&StopFlag)) + return; + GetSide<false>().InputQueue.Push(TSide::TItem(stream, std::move(operation))); + } + + void + TPollerUnit::StartWriteOperation( const TIntrusivePtr<TSharedDescriptor>& stream, - TFDDelegate&& operation) { - Y_VERIFY_DEBUG(stream); - if (AtomicLoad(&StopFlag)) - return; - GetSide<true>().InputQueue.Push(TSide::TItem(stream, std::move(operation))); - } - - template <bool IsWrite> - void* - TPollerUnit::IdleThread(void* param) { + TFDDelegate&& operation) { + Y_VERIFY_DEBUG(stream); + if (AtomicLoad(&StopFlag)) + return; + GetSide<true>().InputQueue.Push(TSide::TItem(stream, std::move(operation))); + } + + template <bool IsWrite> + void* + TPollerUnit::IdleThread(void* param) { // TODO: musl-libc version of `sched_param` struct is for some reason different from pthread // version in Ubuntu 12.04 #if defined(_linux_) && !defined(_musl_) - pthread_t threadSelf = pthread_self(); - sched_param sparam = {20}; - pthread_setschedparam(threadSelf, SCHED_FIFO, &sparam); + pthread_t threadSelf = pthread_self(); + sched_param sparam = {20}; + pthread_setschedparam(threadSelf, SCHED_FIFO, &sparam); #endif - static_cast<TPollerUnit*>(param)->RunLoop<IsWrite>(); - return nullptr; - } + static_cast<TPollerUnit*>(param)->RunLoop<IsWrite>(); + return nullptr; + } - template <> - void - TPollerUnit::RunLoop<false>() { + template <> + void + TPollerUnit::RunLoop<false>() { NProfiling::TMemoryTagScope tag("INTERCONNECT_RECEIVED_DATA"); - while (!AtomicLoad(&StopFlag)) - ProcessRead(); - } + while (!AtomicLoad(&StopFlag)) + ProcessRead(); + } - template <> - void - TPollerUnit::RunLoop<true>() { + template <> + void + TPollerUnit::RunLoop<true>() { NProfiling::TMemoryTagScope tag("INTERCONNECT_SEND_DATA"); - while (!AtomicLoad(&StopFlag)) - ProcessWrite(); - } - - void - TPollerUnit::TSide::ProcessInput() { - if (!InputQueue.IsEmpty()) - do { - auto sock = InputQueue.Top().first->GetDescriptor(); - if (!Operations.emplace(sock, std::move(InputQueue.Top())).second) - Y_FAIL("Descriptor is already in pooler."); - } while (InputQueue.Pop()); - } + while (!AtomicLoad(&StopFlag)) + ProcessWrite(); + } + + void + TPollerUnit::TSide::ProcessInput() { + if (!InputQueue.IsEmpty()) + do { + auto sock = InputQueue.Top().first->GetDescriptor(); + if (!Operations.emplace(sock, std::move(InputQueue.Top())).second) + Y_FAIL("Descriptor is already in pooler."); + } while (InputQueue.Pop()); + } } diff --git a/library/cpp/actors/interconnect/poller_tcp_unit.h b/library/cpp/actors/interconnect/poller_tcp_unit.h index 692168b968..d3f0d0fd0a 100644 --- a/library/cpp/actors/interconnect/poller_tcp_unit.h +++ b/library/cpp/actors/interconnect/poller_tcp_unit.h @@ -10,58 +10,58 @@ #include <unordered_map> namespace NInterconnect { - using NActors::TFDDelegate; - using NActors::TSharedDescriptor; + using NActors::TFDDelegate; + using NActors::TSharedDescriptor; - class TPollerUnit { - public: - typedef std::unique_ptr<TPollerUnit> TPtr; + class TPollerUnit { + public: + typedef std::unique_ptr<TPollerUnit> TPtr; - static TPtr Make(bool useSelect); + static TPtr Make(bool useSelect); - void Start(); - void Stop(); + void Start(); + void Stop(); - virtual void StartReadOperation( - const TIntrusivePtr<TSharedDescriptor>& stream, - TFDDelegate&& operation); + virtual void StartReadOperation( + const TIntrusivePtr<TSharedDescriptor>& stream, + TFDDelegate&& operation); - virtual void StartWriteOperation( - const TIntrusivePtr<TSharedDescriptor>& stream, - TFDDelegate&& operation); + virtual void StartWriteOperation( + const TIntrusivePtr<TSharedDescriptor>& stream, + TFDDelegate&& operation); - virtual ~TPollerUnit(); + virtual ~TPollerUnit(); - private: - virtual void ProcessRead() = 0; - virtual void ProcessWrite() = 0; + private: + virtual void ProcessRead() = 0; + virtual void ProcessWrite() = 0; - template <bool IsWrite> - static void* IdleThread(void* param); + template <bool IsWrite> + static void* IdleThread(void* param); - template <bool IsWrite> - void RunLoop(); + template <bool IsWrite> + void RunLoop(); - volatile bool StopFlag; - TThread ReadLoop, WriteLoop; + volatile bool StopFlag; + TThread ReadLoop, WriteLoop; - protected: - TPollerUnit(); + protected: + TPollerUnit(); - struct TSide { - using TOperations = - std::unordered_map<SOCKET, - std::pair<TIntrusivePtr<TSharedDescriptor>, TFDDelegate>>; + struct TSide { + using TOperations = + std::unordered_map<SOCKET, + std::pair<TIntrusivePtr<TSharedDescriptor>, TFDDelegate>>; - TOperations Operations; - using TItem = TOperations::mapped_type; - TFunnelQueue<TItem> InputQueue; + TOperations Operations; + using TItem = TOperations::mapped_type; + TFunnelQueue<TItem> InputQueue; - void ProcessInput(); - } Read, Write; + void ProcessInput(); + } Read, Write; - template <bool IsWrite> - TSide& GetSide(); - }; + template <bool IsWrite> + TSide& GetSide(); + }; } diff --git a/library/cpp/actors/interconnect/poller_tcp_unit_epoll.cpp b/library/cpp/actors/interconnect/poller_tcp_unit_epoll.cpp index c78538b95b..c0c4524f1e 100644 --- a/library/cpp/actors/interconnect/poller_tcp_unit_epoll.cpp +++ b/library/cpp/actors/interconnect/poller_tcp_unit_epoll.cpp @@ -8,117 +8,117 @@ #include <cstring> namespace NInterconnect { - namespace { - void - DeleteEpoll(int epoll, SOCKET stream) { - ::epoll_event event = {0, {.fd = stream}}; - if (::epoll_ctl(epoll, EPOLL_CTL_DEL, stream, &event)) { - Cerr << "epoll_ctl errno: " << errno << Endl; - Y_FAIL("epoll delete error!"); - } - } - - template <ui32 Events> - void - AddEpoll(int epoll, SOCKET stream) { + namespace { + void + DeleteEpoll(int epoll, SOCKET stream) { + ::epoll_event event = {0, {.fd = stream}}; + if (::epoll_ctl(epoll, EPOLL_CTL_DEL, stream, &event)) { + Cerr << "epoll_ctl errno: " << errno << Endl; + Y_FAIL("epoll delete error!"); + } + } + + template <ui32 Events> + void + AddEpoll(int epoll, SOCKET stream) { ::epoll_event event = {.events = Events}; - event.data.fd = stream; - if (::epoll_ctl(epoll, EPOLL_CTL_ADD, stream, &event)) { - Cerr << "epoll_ctl errno: " << errno << Endl; - Y_FAIL("epoll add error!"); - } - } - - int - Initialize() { - const auto epoll = ::epoll_create(10000); - Y_VERIFY_DEBUG(epoll > 0); - return epoll; - } - + event.data.fd = stream; + if (::epoll_ctl(epoll, EPOLL_CTL_ADD, stream, &event)) { + Cerr << "epoll_ctl errno: " << errno << Endl; + Y_FAIL("epoll add error!"); + } + } + + int + Initialize() { + const auto epoll = ::epoll_create(10000); + Y_VERIFY_DEBUG(epoll > 0); + return epoll; + } + } - TPollerUnitEpoll::TPollerUnitEpoll() - : ReadDescriptor(Initialize()) - , WriteDescriptor(Initialize()) - { - // Block on the epoll descriptor. - ::sigemptyset(&sigmask); - ::sigaddset(&sigmask, SIGPIPE); - ::sigaddset(&sigmask, SIGTERM); + TPollerUnitEpoll::TPollerUnitEpoll() + : ReadDescriptor(Initialize()) + , WriteDescriptor(Initialize()) + { + // Block on the epoll descriptor. + ::sigemptyset(&sigmask); + ::sigaddset(&sigmask, SIGPIPE); + ::sigaddset(&sigmask, SIGTERM); } - TPollerUnitEpoll::~TPollerUnitEpoll() { - ::close(ReadDescriptor); - ::close(WriteDescriptor); - } + TPollerUnitEpoll::~TPollerUnitEpoll() { + ::close(ReadDescriptor); + ::close(WriteDescriptor); + } - template <> - int TPollerUnitEpoll::GetDescriptor<false>() const { - return ReadDescriptor; - } + template <> + int TPollerUnitEpoll::GetDescriptor<false>() const { + return ReadDescriptor; + } - template <> - int TPollerUnitEpoll::GetDescriptor<true>() const { - return WriteDescriptor; - } + template <> + int TPollerUnitEpoll::GetDescriptor<true>() const { + return WriteDescriptor; + } - void - TPollerUnitEpoll::StartReadOperation( + void + TPollerUnitEpoll::StartReadOperation( const TIntrusivePtr<TSharedDescriptor>& s, - TFDDelegate&& operation) { - TPollerUnit::StartReadOperation(s, std::move(operation)); - AddEpoll<EPOLLRDHUP | EPOLLIN>(ReadDescriptor, s->GetDescriptor()); - } + TFDDelegate&& operation) { + TPollerUnit::StartReadOperation(s, std::move(operation)); + AddEpoll<EPOLLRDHUP | EPOLLIN>(ReadDescriptor, s->GetDescriptor()); + } - void - TPollerUnitEpoll::StartWriteOperation( + void + TPollerUnitEpoll::StartWriteOperation( const TIntrusivePtr<TSharedDescriptor>& s, - TFDDelegate&& operation) { - TPollerUnit::StartWriteOperation(s, std::move(operation)); - AddEpoll<EPOLLRDHUP | EPOLLOUT>(WriteDescriptor, s->GetDescriptor()); - } - - constexpr int EVENTS_BUF_SIZE = 128; - - template <bool WriteOp> - void - TPollerUnitEpoll::Process() { - ::epoll_event events[EVENTS_BUF_SIZE]; - - const int epoll = GetDescriptor<WriteOp>(); - - /* Timeout just to check StopFlag sometimes */ - const int result = - ::epoll_pwait(epoll, events, EVENTS_BUF_SIZE, 200, &sigmask); - - if (result == -1 && errno != EINTR) - Y_FAIL("epoll wait error!"); - - auto& side = GetSide<WriteOp>(); - side.ProcessInput(); - - for (int i = 0; i < result; ++i) { - const auto it = side.Operations.find(events[i].data.fd); - if (side.Operations.end() == it) - continue; - if (const auto& finalizer = it->second.second(it->second.first)) { - DeleteEpoll(epoll, it->first); - side.Operations.erase(it); - finalizer(); - } + TFDDelegate&& operation) { + TPollerUnit::StartWriteOperation(s, std::move(operation)); + AddEpoll<EPOLLRDHUP | EPOLLOUT>(WriteDescriptor, s->GetDescriptor()); + } + + constexpr int EVENTS_BUF_SIZE = 128; + + template <bool WriteOp> + void + TPollerUnitEpoll::Process() { + ::epoll_event events[EVENTS_BUF_SIZE]; + + const int epoll = GetDescriptor<WriteOp>(); + + /* Timeout just to check StopFlag sometimes */ + const int result = + ::epoll_pwait(epoll, events, EVENTS_BUF_SIZE, 200, &sigmask); + + if (result == -1 && errno != EINTR) + Y_FAIL("epoll wait error!"); + + auto& side = GetSide<WriteOp>(); + side.ProcessInput(); + + for (int i = 0; i < result; ++i) { + const auto it = side.Operations.find(events[i].data.fd); + if (side.Operations.end() == it) + continue; + if (const auto& finalizer = it->second.second(it->second.first)) { + DeleteEpoll(epoll, it->first); + side.Operations.erase(it); + finalizer(); + } } } - void - TPollerUnitEpoll::ProcessRead() { - Process<false>(); - } + void + TPollerUnitEpoll::ProcessRead() { + Process<false>(); + } - void - TPollerUnitEpoll::ProcessWrite() { - Process<true>(); - } + void + TPollerUnitEpoll::ProcessWrite() { + Process<true>(); + } } diff --git a/library/cpp/actors/interconnect/poller_tcp_unit_epoll.h b/library/cpp/actors/interconnect/poller_tcp_unit_epoll.h index ff7893eba2..70aee1c66b 100644 --- a/library/cpp/actors/interconnect/poller_tcp_unit_epoll.h +++ b/library/cpp/actors/interconnect/poller_tcp_unit_epoll.h @@ -3,31 +3,31 @@ #include "poller_tcp_unit.h" namespace NInterconnect { - class TPollerUnitEpoll: public TPollerUnit { - public: - TPollerUnitEpoll(); - virtual ~TPollerUnitEpoll(); + class TPollerUnitEpoll: public TPollerUnit { + public: + TPollerUnitEpoll(); + virtual ~TPollerUnitEpoll(); - private: - virtual void StartReadOperation( - const TIntrusivePtr<TSharedDescriptor>& s, - TFDDelegate&& operation) override; + private: + virtual void StartReadOperation( + const TIntrusivePtr<TSharedDescriptor>& s, + TFDDelegate&& operation) override; - virtual void StartWriteOperation( - const TIntrusivePtr<TSharedDescriptor>& s, - TFDDelegate&& operation) override; + virtual void StartWriteOperation( + const TIntrusivePtr<TSharedDescriptor>& s, + TFDDelegate&& operation) override; - virtual void ProcessRead() override; - virtual void ProcessWrite() override; + virtual void ProcessRead() override; + virtual void ProcessWrite() override; - template <bool Write> - void Process(); + template <bool Write> + void Process(); - template <bool Write> - int GetDescriptor() const; + template <bool Write> + int GetDescriptor() const; - const int ReadDescriptor, WriteDescriptor; - ::sigset_t sigmask; - }; + const int ReadDescriptor, WriteDescriptor; + ::sigset_t sigmask; + }; } diff --git a/library/cpp/actors/interconnect/poller_tcp_unit_select.cpp b/library/cpp/actors/interconnect/poller_tcp_unit_select.cpp index ae7aaad566..f0acc6d338 100644 --- a/library/cpp/actors/interconnect/poller_tcp_unit_select.cpp +++ b/library/cpp/actors/interconnect/poller_tcp_unit_select.cpp @@ -15,72 +15,72 @@ typedef timeval TIMEVAL; #endif namespace NInterconnect { - TPollerUnitSelect::TPollerUnitSelect() { - } + TPollerUnitSelect::TPollerUnitSelect() { + } - TPollerUnitSelect::~TPollerUnitSelect() { - } + TPollerUnitSelect::~TPollerUnitSelect() { + } - template <bool IsWrite> - void - TPollerUnitSelect::Process() { - auto& side = GetSide<IsWrite>(); - side.ProcessInput(); + template <bool IsWrite> + void + TPollerUnitSelect::Process() { + auto& side = GetSide<IsWrite>(); + side.ProcessInput(); - enum : size_t { R, - W, - E }; - static const auto O = IsWrite ? W : R; + enum : size_t { R, + W, + E }; + static const auto O = IsWrite ? W : R; - ::fd_set sets[3]; + ::fd_set sets[3]; - FD_ZERO(&sets[R]); - FD_ZERO(&sets[W]); - FD_ZERO(&sets[E]); + FD_ZERO(&sets[R]); + FD_ZERO(&sets[W]); + FD_ZERO(&sets[E]); - for (const auto& operation : side.Operations) { - FD_SET(operation.first, &sets[O]); - FD_SET(operation.first, &sets[E]); - } + for (const auto& operation : side.Operations) { + FD_SET(operation.first, &sets[O]); + FD_SET(operation.first, &sets[E]); + } #if defined(_win_) - ::TIMEVAL timeout = {0L, 99991L}; - const auto numberEvents = !side.Operations.empty() ? ::select(FD_SETSIZE, &sets[R], &sets[W], &sets[E], &timeout) - : (::Sleep(100), 0); + ::TIMEVAL timeout = {0L, 99991L}; + const auto numberEvents = !side.Operations.empty() ? ::select(FD_SETSIZE, &sets[R], &sets[W], &sets[E], &timeout) + : (::Sleep(100), 0); #elif defined(_darwin_) - ::TIMEVAL timeout = {0L, 99991L}; - const auto numberEvents = ::select(FD_SETSIZE, &sets[R], &sets[W], &sets[E], &timeout); + ::TIMEVAL timeout = {0L, 99991L}; + const auto numberEvents = ::select(FD_SETSIZE, &sets[R], &sets[W], &sets[E], &timeout); #else - ::sigset_t sigmask; - ::sigemptyset(&sigmask); - ::sigaddset(&sigmask, SIGPIPE); - ::sigaddset(&sigmask, SIGTERM); + ::sigset_t sigmask; + ::sigemptyset(&sigmask); + ::sigaddset(&sigmask, SIGPIPE); + ::sigaddset(&sigmask, SIGTERM); - struct ::timespec timeout = {0L, 99999989L}; - const auto numberEvents = ::pselect(FD_SETSIZE, &sets[R], &sets[W], &sets[E], &timeout, &sigmask); + struct ::timespec timeout = {0L, 99999989L}; + const auto numberEvents = ::pselect(FD_SETSIZE, &sets[R], &sets[W], &sets[E], &timeout, &sigmask); #endif - Y_VERIFY_DEBUG(numberEvents >= 0); - - for (auto it = side.Operations.cbegin(); side.Operations.cend() != it;) { - if (FD_ISSET(it->first, &sets[O]) || FD_ISSET(it->first, &sets[E])) - if (const auto& finalizer = it->second.second(it->second.first)) { - side.Operations.erase(it++); - finalizer(); - continue; - } - ++it; - } + Y_VERIFY_DEBUG(numberEvents >= 0); + + for (auto it = side.Operations.cbegin(); side.Operations.cend() != it;) { + if (FD_ISSET(it->first, &sets[O]) || FD_ISSET(it->first, &sets[E])) + if (const auto& finalizer = it->second.second(it->second.first)) { + side.Operations.erase(it++); + finalizer(); + continue; + } + ++it; + } } - void - TPollerUnitSelect::ProcessRead() { - Process<false>(); - } + void + TPollerUnitSelect::ProcessRead() { + Process<false>(); + } - void - TPollerUnitSelect::ProcessWrite() { - Process<true>(); - } + void + TPollerUnitSelect::ProcessWrite() { + Process<true>(); + } } diff --git a/library/cpp/actors/interconnect/poller_tcp_unit_select.h b/library/cpp/actors/interconnect/poller_tcp_unit_select.h index 0c15217796..7b6d5b3996 100644 --- a/library/cpp/actors/interconnect/poller_tcp_unit_select.h +++ b/library/cpp/actors/interconnect/poller_tcp_unit_select.h @@ -3,17 +3,17 @@ #include "poller_tcp_unit.h" namespace NInterconnect { - class TPollerUnitSelect: public TPollerUnit { - public: - TPollerUnitSelect(); - virtual ~TPollerUnitSelect(); + class TPollerUnitSelect: public TPollerUnit { + public: + TPollerUnitSelect(); + virtual ~TPollerUnitSelect(); - private: - virtual void ProcessRead() override; - virtual void ProcessWrite() override; + private: + virtual void ProcessRead() override; + virtual void ProcessWrite() override; - template <bool IsWrite> - void Process(); - }; + template <bool IsWrite> + void Process(); + }; } diff --git a/library/cpp/actors/interconnect/ut/lib/ic_test_cluster.h b/library/cpp/actors/interconnect/ut/lib/ic_test_cluster.h index 2b6d27cd3f..d17755cad1 100644 --- a/library/cpp/actors/interconnect/ut/lib/ic_test_cluster.h +++ b/library/cpp/actors/interconnect/ut/lib/ic_test_cluster.h @@ -9,14 +9,14 @@ #include <util/generic/noncopyable.h> -class TTestICCluster: public TNonCopyable { +class TTestICCluster: public TNonCopyable { public: struct TTrafficInterrupterSettings { TDuration RejectingTrafficTimeout; double BandWidth; bool Disconnect; }; - + private: const ui32 NumNodes; const TString Address = "::1"; @@ -29,7 +29,7 @@ private: public: TTestICCluster(ui32 numNodes = 1, NActors::TChannelsConfig channelsConfig = NActors::TChannelsConfig(), - TTrafficInterrupterSettings* tiSettings = nullptr) + TTrafficInterrupterSettings* tiSettings = nullptr) : NumNodes(numNodes) , Counters(new NMonitoring::TDynamicCounters) , ChannelsConfig(channelsConfig) @@ -67,7 +67,7 @@ public: return Nodes[id].Get(); } - ~TTestICCluster() { + ~TTestICCluster() { } TActorId RegisterActor(NActors::IActor* actor, ui32 nodeId) { diff --git a/library/cpp/actors/interconnect/ut/lib/interrupter.h b/library/cpp/actors/interconnect/ut/lib/interrupter.h index 48851de2c5..a4a4373f3e 100644 --- a/library/cpp/actors/interconnect/ut/lib/interrupter.h +++ b/library/cpp/actors/interconnect/ut/lib/interrupter.h @@ -26,7 +26,7 @@ class TTrafficInterrupter TVector<char> Data; }; struct TCompare { - bool operator()(const std::pair<TInstant, TDelayedPacket>& x, const std::pair<TInstant, TDelayedPacket>& y) const { + bool operator()(const std::pair<TInstant, TDelayedPacket>& x, const std::pair<TInstant, TDelayedPacket>& y) const { return x.first > y.first; }; }; @@ -160,7 +160,7 @@ private: timeout = Min(timeout, conn.DelayedQueue.top().first - TInstant::Now()); } }; - for (auto& it : Connections) { + for (auto& it : Connections) { updateTimout(it.ForwardConnection); updateTimout(it.BackwardConnection); } @@ -177,20 +177,20 @@ private: } if (DelayTraffic) { // process packets from DelayQueues auto processDelayedPackages = [](TDirectedConnection& conn) { - while (!conn.DelayedQueue.empty()) { - auto& frontPackage = conn.DelayedQueue.top(); - if (TInstant::Now() >= frontPackage.first) { - TInet6StreamSocket* sock = frontPackage.second.ForwardSocket; - if (sock) { + while (!conn.DelayedQueue.empty()) { + auto& frontPackage = conn.DelayedQueue.top(); + if (TInstant::Now() >= frontPackage.first) { + TInet6StreamSocket* sock = frontPackage.second.ForwardSocket; + if (sock) { sock->Send(frontPackage.second.Data.data(), frontPackage.second.Data.size()); } - conn.DelayedQueue.pop(); - } else { - break; + conn.DelayedQueue.pop(); + } else { + break; } - } - }; - for (auto& it : Connections) { + } + }; + for (auto& it : Connections) { processDelayedPackages(it.ForwardConnection); processDelayedPackages(it.BackwardConnection); } @@ -228,7 +228,7 @@ private: if (recvSize > 0) { if (DelayTraffic) { // put packet into DelayQueue - const TDuration baseDelay = TDuration::MicroSeconds(recvSize * 1e6 / Bandwidth); + const TDuration baseDelay = TDuration::MicroSeconds(recvSize * 1e6 / Bandwidth); const TInstant now = TInstant::Now(); directedConnection->Timestamp = Max(now, directedConnection->Timestamp) + baseDelay; TDelayedPacket pkt; diff --git a/library/cpp/actors/interconnect/ut/lib/node.h b/library/cpp/actors/interconnect/ut/lib/node.h index ff30b1445e..cd3a39a6c0 100644 --- a/library/cpp/actors/interconnect/ut/lib/node.h +++ b/library/cpp/actors/interconnect/ut/lib/node.h @@ -63,7 +63,7 @@ public: TMailboxType::ReadAsFilled, 0)); const TActorId loggerActorId(0, "logger"); - constexpr ui32 LoggerComponentId = 410; // NKikimrServices::LOGGER + constexpr ui32 LoggerComponentId = 410; // NKikimrServices::LOGGER auto loggerSettings = MakeIntrusive<NLog::TSettings>( loggerActorId, @@ -78,7 +78,7 @@ public: NActorsServices::EServiceCommon_Name ); - constexpr ui32 WilsonComponentId = 430; // NKikimrServices::WILSON + constexpr ui32 WilsonComponentId = 430; // NKikimrServices::WILSON static const TString WilsonComponentName = "WILSON"; loggerSettings->Append( diff --git a/library/cpp/actors/interconnect/ut/lib/test_actors.h b/library/cpp/actors/interconnect/ut/lib/test_actors.h index 7591200471..bcb3504ebe 100644 --- a/library/cpp/actors/interconnect/ut/lib/test_actors.h +++ b/library/cpp/actors/interconnect/ut/lib/test_actors.h @@ -1,56 +1,56 @@ #pragma once namespace NActors { - class TSenderBaseActor: public TActorBootstrapped<TSenderBaseActor> { - protected: + class TSenderBaseActor: public TActorBootstrapped<TSenderBaseActor> { + protected: const TActorId RecipientActorId; - const ui32 Preload; - ui64 SequenceNumber = 0; - ui32 InFlySize = 0; + const ui32 Preload; + ui64 SequenceNumber = 0; + ui32 InFlySize = 0; - public: + public: TSenderBaseActor(const TActorId& recipientActorId, ui32 preload = 1) - : RecipientActorId(recipientActorId) - , Preload(preload) - { - } + : RecipientActorId(recipientActorId) + , Preload(preload) + { + } - virtual ~TSenderBaseActor() { - } + virtual ~TSenderBaseActor() { + } - virtual void Bootstrap(const TActorContext& ctx) { - Become(&TSenderBaseActor::StateFunc); + virtual void Bootstrap(const TActorContext& ctx) { + Become(&TSenderBaseActor::StateFunc); ctx.Send(ctx.ExecutorThread.ActorSystem->InterconnectProxy(RecipientActorId.NodeId()), new TEvInterconnect::TEvConnectNode); - } + } - virtual void SendMessagesIfPossible(const TActorContext& ctx) { - while (InFlySize < Preload) { - SendMessage(ctx); - } + virtual void SendMessagesIfPossible(const TActorContext& ctx) { + while (InFlySize < Preload) { + SendMessage(ctx); + } } - virtual void SendMessage(const TActorContext& /*ctx*/) { - ++SequenceNumber; - } + virtual void SendMessage(const TActorContext& /*ctx*/) { + ++SequenceNumber; + } - virtual void Handle(TEvents::TEvUndelivered::TPtr& /*ev*/, const TActorContext& ctx) { - SendMessage(ctx); - } + virtual void Handle(TEvents::TEvUndelivered::TPtr& /*ev*/, const TActorContext& ctx) { + SendMessage(ctx); + } - virtual void Handle(TEvTestResponse::TPtr& /*ev*/, const TActorContext& ctx) { - SendMessagesIfPossible(ctx); - } + virtual void Handle(TEvTestResponse::TPtr& /*ev*/, const TActorContext& ctx) { + SendMessagesIfPossible(ctx); + } void Handle(TEvInterconnect::TEvNodeConnected::TPtr& /*ev*/, const TActorContext& ctx) { SendMessagesIfPossible(ctx); - } + } - void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& /*ev*/, const TActorContext& /*ctx*/) { - } + void Handle(TEvInterconnect::TEvNodeDisconnected::TPtr& /*ev*/, const TActorContext& /*ctx*/) { + } - virtual void Handle(TEvents::TEvPoisonPill::TPtr& /*ev*/, const TActorContext& ctx) { - Die(ctx); - } + virtual void Handle(TEvents::TEvPoisonPill::TPtr& /*ev*/, const TActorContext& ctx) { + Die(ctx); + } virtual STRICT_STFUNC(StateFunc, HFunc(TEvTestResponse, Handle) @@ -59,25 +59,25 @@ namespace NActors { HFunc(TEvInterconnect::TEvNodeConnected, Handle) HFunc(TEvInterconnect::TEvNodeDisconnected, Handle) ) - }; + }; - class TReceiverBaseActor: public TActor<TReceiverBaseActor> { - protected: - ui64 ReceivedCount = 0; + class TReceiverBaseActor: public TActor<TReceiverBaseActor> { + protected: + ui64 ReceivedCount = 0; - public: - TReceiverBaseActor() - : TActor(&TReceiverBaseActor::StateFunc) - { - } + public: + TReceiverBaseActor() + : TActor(&TReceiverBaseActor::StateFunc) + { + } - virtual ~TReceiverBaseActor() { - } + virtual ~TReceiverBaseActor() { + } virtual STRICT_STFUNC(StateFunc, HFunc(TEvTest, Handle) ) virtual void Handle(TEvTest::TPtr& /*ev*/, const TActorContext& /*ctx*/) {} - }; + }; } diff --git a/library/cpp/actors/interconnect/ut/lib/test_events.h b/library/cpp/actors/interconnect/ut/lib/test_events.h index cd0d9e0152..29d52b20af 100644 --- a/library/cpp/actors/interconnect/ut/lib/test_events.h +++ b/library/cpp/actors/interconnect/ut/lib/test_events.h @@ -2,7 +2,7 @@ #include <library/cpp/actors/interconnect/ut/protos/interconnect_test.pb.h> -namespace NActors { +namespace NActors { enum { EvTest = EventSpaceBegin(TEvents::ES_PRIVATE), EvTestChan, diff --git a/library/cpp/actors/interconnect/ut_fat/main.cpp b/library/cpp/actors/interconnect/ut_fat/main.cpp index 5d19bc3003..c83f7c2ddf 100644 --- a/library/cpp/actors/interconnect/ut_fat/main.cpp +++ b/library/cpp/actors/interconnect/ut_fat/main.cpp @@ -61,13 +61,13 @@ Y_UNIT_TEST_SUITE(InterconnectUnstableConnection) { const NInterconnectTest::TEvTestResponse& record = ev->Get()->Record; Y_VERIFY(record.HasConfirmedSequenceNumber()); if (!(SendFlags & IEventHandle::FlagGenerateUnsureUndelivered)) { - while (record.GetConfirmedSequenceNumber() != InFly.front()) { + while (record.GetConfirmedSequenceNumber() != InFly.front()) { InFly.pop_front(); --InFlySize; } } Y_VERIFY(record.GetConfirmedSequenceNumber() == InFly.front(), "got# %" PRIu64 " expected# %" PRIu64, - record.GetConfirmedSequenceNumber(), InFly.front()); + record.GetConfirmedSequenceNumber(), InFly.front()); InFly.pop_front(); --InFlySize; SendMessagesIfPossible(ctx); diff --git a/library/cpp/actors/interconnect/watchdog_timer.h b/library/cpp/actors/interconnect/watchdog_timer.h index c190105a59..52376aa3fb 100644 --- a/library/cpp/actors/interconnect/watchdog_timer.h +++ b/library/cpp/actors/interconnect/watchdog_timer.h @@ -1,7 +1,7 @@ #pragma once namespace NActors { - template <typename TEvent> + template <typename TEvent> class TWatchdogTimer { using TCallback = std::function<void()>; @@ -9,7 +9,7 @@ namespace NActors { const TCallback Callback; TInstant LastResetTimestamp; - TEvent* ExpectedEvent = nullptr; + TEvent* ExpectedEvent = nullptr; ui32 Iteration = 0; static constexpr ui32 NumIterationsBeforeFiring = 2; @@ -18,8 +18,8 @@ namespace NActors { TWatchdogTimer(TDuration timeout, TCallback callback) : Timeout(timeout) , Callback(std::move(callback)) - { - } + { + } void Arm(const TActorIdentity& actor) { if (Timeout != TDuration::Zero() && Timeout != TDuration::Max()) { @@ -65,4 +65,4 @@ namespace NActors { } }; -} +} diff --git a/library/cpp/actors/memory_log/memlog.cpp b/library/cpp/actors/memory_log/memlog.cpp index 8e6b46727d..14ebb3db25 100644 --- a/library/cpp/actors/memory_log/memlog.cpp +++ b/library/cpp/actors/memory_log/memlog.cpp @@ -23,41 +23,41 @@ static int (*FastGetCpu)(unsigned* cpu, unsigned* node, void* unused); #endif const char TMemoryLog::DEFAULT_LAST_MARK[16] = { - 'c', - 'b', - '7', - 'B', - '6', - '8', - 'a', - '8', - 'A', - '5', - '6', - '1', - '6', - '4', - '5', - '\n', + 'c', + 'b', + '7', + 'B', + '6', + '8', + 'a', + '8', + 'A', + '5', + '6', + '1', + '6', + '4', + '5', + '\n', }; const char TMemoryLog::CLEAR_MARK[16] = { - ' ', - ' ', - ' ', - ' ', - ' ', - ' ', - ' ', - ' ', - ' ', - ' ', - ' ', - ' ', - ' ', - ' ', - ' ', - '\n', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + ' ', + '\n', }; unsigned TMemoryLog::GetSelfCpu() noexcept { diff --git a/library/cpp/actors/prof/tag.cpp b/library/cpp/actors/prof/tag.cpp index 9ccf03e1a9..5667b27dad 100644 --- a/library/cpp/actors/prof/tag.cpp +++ b/library/cpp/actors/prof/tag.cpp @@ -116,4 +116,4 @@ namespace NProfiling { #endif TSetThreadAllocTag* SetThreadAllocTag = SetThreadAllocTagFn(); -} +} diff --git a/library/cpp/actors/prof/tag.h b/library/cpp/actors/prof/tag.h index 357e264a22..9a1f2a2350 100644 --- a/library/cpp/actors/prof/tag.h +++ b/library/cpp/actors/prof/tag.h @@ -70,4 +70,4 @@ namespace NProfiling { ui32 RestoreTag = 0; bool Released = false; }; -} +} diff --git a/library/cpp/actors/testlib/test_runtime.cpp b/library/cpp/actors/testlib/test_runtime.cpp index 6fa25b9965..ca644a5ff7 100644 --- a/library/cpp/actors/testlib/test_runtime.cpp +++ b/library/cpp/actors/testlib/test_runtime.cpp @@ -1860,7 +1860,7 @@ namespace NActors { const TActorId Delegatee; const bool IsSync; const TVector<TActorId> AdditionalActors; - TSimpleSharedPtr<TStrandingActorDecoratorContext> Context; + TSimpleSharedPtr<TStrandingActorDecoratorContext> Context; TActorId ReplyId; bool HasReply; TDispatchOptions DelegateeOptions; @@ -1888,7 +1888,7 @@ namespace NActors { } private: - TSimpleSharedPtr<TStrandingActorDecoratorContext> Context; + TSimpleSharedPtr<TStrandingActorDecoratorContext> Context; TTestActorRuntimeBase* Runtime; TReplyCheckerCreator CreateReplyChecker; }; diff --git a/library/cpp/actors/util/funnel_queue.h b/library/cpp/actors/util/funnel_queue.h index 0e21e2617c..855ed4bee1 100644 --- a/library/cpp/actors/util/funnel_queue.h +++ b/library/cpp/actors/util/funnel_queue.h @@ -3,33 +3,33 @@ #include <util/system/atomic.h> #include <util/generic/noncopyable.h> -template <typename ElementType> -class TFunnelQueue: private TNonCopyable { +template <typename ElementType> +class TFunnelQueue: private TNonCopyable { public: TFunnelQueue() noexcept - : Front(nullptr) - , Back(nullptr) - { - } + : Front(nullptr) + , Back(nullptr) + { + } - virtual ~TFunnelQueue() noexcept { + virtual ~TFunnelQueue() noexcept { for (auto entry = Front; entry; entry = DeleteEntry(entry)) continue; } /// Push element. Can be used from many threads. Return true if is first element. bool - Push(ElementType&& element) noexcept { - TEntry* const next = NewEntry(static_cast<ElementType&&>(element)); - TEntry* const prev = AtomicSwap(&Back, next); + Push(ElementType&& element) noexcept { + TEntry* const next = NewEntry(static_cast<ElementType&&>(element)); + TEntry* const prev = AtomicSwap(&Back, next); AtomicSet(prev ? prev->Next : Front, next); return !prev; } /// Extract top element. Must be used only from one thread. Return true if have more. bool - Pop() noexcept { - if (TEntry* const top = AtomicGet(Front)) { + Pop() noexcept { + if (TEntry* const top = AtomicGet(Front)) { const auto last = AtomicCas(&Back, nullptr, top); if (last) // This is last element in queue. Queue is empty now. AtomicCas(&Front, nullptr, top); @@ -51,42 +51,42 @@ public: /// Peek top element. Must be used only from one thread. ElementType& - Top() const noexcept { + Top() const noexcept { return AtomicGet(Front)->Data; } bool - IsEmpty() const noexcept { + IsEmpty() const noexcept { return !AtomicGet(Front); } protected: - class TEntry: private TNonCopyable { - friend class TFunnelQueue; - + class TEntry: private TNonCopyable { + friend class TFunnelQueue; + private: explicit TEntry(ElementType&& element) noexcept - : Data(static_cast<ElementType&&>(element)) - , Next(nullptr) - { - } + : Data(static_cast<ElementType&&>(element)) + , Next(nullptr) + { + } - ~TEntry() noexcept { - } + ~TEntry() noexcept { + } public: ElementType Data; - TEntry* volatile Next; + TEntry* volatile Next; }; - TEntry* volatile Front; - TEntry* volatile Back; + TEntry* volatile Front; + TEntry* volatile Back; - virtual TEntry* NewEntry(ElementType&& element) noexcept { + virtual TEntry* NewEntry(ElementType&& element) noexcept { return new TEntry(static_cast<ElementType&&>(element)); } - virtual TEntry* DeleteEntry(TEntry* entry) noexcept { + virtual TEntry* DeleteEntry(TEntry* entry) noexcept { const auto next = entry->Next; delete entry; return next; @@ -147,36 +147,36 @@ public: using const_iterator = TConstEntryIter; using iterator = TEntryIter; - iterator begin() { - return {AtomicGet(Front)}; - } - const_iterator cbegin() { - return {AtomicGet(Front)}; - } - const_iterator begin() const { - return {AtomicGet(Front)}; - } - - iterator end() { - return {nullptr}; - } - const_iterator cend() { - return {nullptr}; - } - const_iterator end() const { - return {nullptr}; - } + iterator begin() { + return {AtomicGet(Front)}; + } + const_iterator cbegin() { + return {AtomicGet(Front)}; + } + const_iterator begin() const { + return {AtomicGet(Front)}; + } + + iterator end() { + return {nullptr}; + } + const_iterator cend() { + return {nullptr}; + } + const_iterator end() const { + return {nullptr}; + } }; -template <typename ElementType> -class TPooledFunnelQueue: public TFunnelQueue<ElementType> { +template <typename ElementType> +class TPooledFunnelQueue: public TFunnelQueue<ElementType> { public: TPooledFunnelQueue() noexcept : Stack(nullptr) - { - } + { + } - virtual ~TPooledFunnelQueue() noexcept override { + virtual ~TPooledFunnelQueue() noexcept override { for (auto entry = TBase::Front; entry; entry = TBase::DeleteEntry(entry)) continue; for (auto entry = Stack; entry; entry = TBase::DeleteEntry(entry)) @@ -187,10 +187,10 @@ public: private: typedef TFunnelQueue<ElementType> TBase; - typename TBase::TEntry* volatile Stack; + typename TBase::TEntry* volatile Stack; protected: - virtual typename TBase::TEntry* NewEntry(ElementType&& element) noexcept override { + virtual typename TBase::TEntry* NewEntry(ElementType&& element) noexcept override { while (const auto top = AtomicGet(Stack)) if (AtomicCas(&Stack, top->Next, top)) { top->Data = static_cast<ElementType&&>(element); @@ -201,37 +201,37 @@ protected: return TBase::NewEntry(static_cast<ElementType&&>(element)); } - virtual typename TBase::TEntry* DeleteEntry(typename TBase::TEntry* entry) noexcept override { + virtual typename TBase::TEntry* DeleteEntry(typename TBase::TEntry* entry) noexcept override { entry->Data = ElementType(); const auto next = entry->Next; - do - AtomicSet(entry->Next, AtomicGet(Stack)); + do + AtomicSet(entry->Next, AtomicGet(Stack)); while (!AtomicCas(&Stack, entry, entry->Next)); return next; } }; -template <typename ElementType, template <typename T> class TQueueType = TFunnelQueue> -class TCountedFunnelQueue: public TQueueType<ElementType> { +template <typename ElementType, template <typename T> class TQueueType = TFunnelQueue> +class TCountedFunnelQueue: public TQueueType<ElementType> { public: TCountedFunnelQueue() noexcept : Count(0) - { - } + { + } - TAtomicBase GetSize() const noexcept { + TAtomicBase GetSize() const noexcept { return AtomicGet(Count); } private: typedef TQueueType<ElementType> TBase; - virtual typename TBase::TEntry* NewEntry(ElementType&& element) noexcept override { + virtual typename TBase::TEntry* NewEntry(ElementType&& element) noexcept override { AtomicAdd(Count, 1); return TBase::NewEntry(static_cast<ElementType&&>(element)); } - virtual typename TBase::TEntry* DeleteEntry(typename TBase::TEntry* entry) noexcept override { + virtual typename TBase::TEntry* DeleteEntry(typename TBase::TEntry* entry) noexcept override { AtomicSub(Count, 1); return TBase::DeleteEntry(entry); } diff --git a/library/cpp/actors/util/queue_chunk.h b/library/cpp/actors/util/queue_chunk.h index 8a4e02d8cb..b56f0c1d5c 100644 --- a/library/cpp/actors/util/queue_chunk.h +++ b/library/cpp/actors/util/queue_chunk.h @@ -5,7 +5,7 @@ template <typename T, ui32 TSize, typename TDerived> struct TQueueChunkDerived { static const ui32 EntriesCount = (TSize - sizeof(TQueueChunkDerived*)) / sizeof(T); - static_assert(EntriesCount > 0, "expect EntriesCount > 0"); + static_assert(EntriesCount > 0, "expect EntriesCount > 0"); volatile T Entries[EntriesCount]; TDerived* volatile Next; @@ -18,7 +18,7 @@ struct TQueueChunkDerived { template <typename T, ui32 TSize> struct TQueueChunk { static const ui32 EntriesCount = (TSize - sizeof(TQueueChunk*)) / sizeof(T); - static_assert(EntriesCount > 0, "expect EntriesCount > 0"); + static_assert(EntriesCount > 0, "expect EntriesCount > 0"); volatile T Entries[EntriesCount]; TQueueChunk* volatile Next; diff --git a/library/cpp/actors/util/recentwnd.h b/library/cpp/actors/util/recentwnd.h index ba1ede6f29..6135814f50 100644 --- a/library/cpp/actors/util/recentwnd.h +++ b/library/cpp/actors/util/recentwnd.h @@ -1,15 +1,15 @@ #pragma once - + #include <util/generic/deque.h> template <typename TElem, template <typename, typename...> class TContainer = TDeque> class TRecentWnd { public: - TRecentWnd(ui32 wndSize) - : MaxWndSize_(wndSize) - { - } + TRecentWnd(ui32 wndSize) + : MaxWndSize_(wndSize) + { + } void Push(const TElem& elem) { if (Window_.size() == MaxWndSize_) @@ -23,27 +23,27 @@ public: Window_.emplace_back(std::move(elem)); } - TElem& Last() { - return Window_.back(); - } - const TElem& Last() const { - return Window_.back(); - } - bool Full() const { - return Window_.size() == MaxWndSize_; - } - ui64 Size() const { - return Window_.size(); - } + TElem& Last() { + return Window_.back(); + } + const TElem& Last() const { + return Window_.back(); + } + bool Full() const { + return Window_.size() == MaxWndSize_; + } + ui64 Size() const { + return Window_.size(); + } using const_iterator = typename TContainer<TElem>::const_iterator; - const_iterator begin() { - return Window_.begin(); - } - const_iterator end() { - return Window_.end(); - } + const_iterator begin() { + return Window_.begin(); + } + const_iterator end() { + return Window_.end(); + } void Reset(ui32 wndSize = 0) { Window_.clear(); @@ -60,7 +60,7 @@ public: Window_.begin() + Window_.size() - MaxWndSize_); } } - + private: TContainer<TElem> Window_; ui32 MaxWndSize_; diff --git a/library/cpp/actors/util/rope.h b/library/cpp/actors/util/rope.h index f5595efbaa..305a035448 100644 --- a/library/cpp/actors/util/rope.h +++ b/library/cpp/actors/util/rope.h @@ -574,7 +574,7 @@ public: TRope(TString s) { if (s) { Size = s.size(); - s.reserve(32); + s.reserve(32); Chain.PutToEnd(std::move(s)); } } diff --git a/library/cpp/actors/util/threadparkpad.cpp b/library/cpp/actors/util/threadparkpad.cpp index 74069ff15b..471be44031 100644 --- a/library/cpp/actors/util/threadparkpad.cpp +++ b/library/cpp/actors/util/threadparkpad.cpp @@ -6,143 +6,143 @@ #include "futex.h" namespace NActors { - class TThreadParkPad::TImpl { - volatile bool Interrupted; - int Futex; - - public: - TImpl() - : Interrupted(false) - , Futex(0) - { - } - ~TImpl() { - } + class TThreadParkPad::TImpl { + volatile bool Interrupted; + int Futex; + + public: + TImpl() + : Interrupted(false) + , Futex(0) + { + } + ~TImpl() { + } bool Park() noexcept { - __atomic_fetch_sub(&Futex, 1, __ATOMIC_SEQ_CST); - while (__atomic_load_n(&Futex, __ATOMIC_ACQUIRE) == -1) - SysFutex(&Futex, FUTEX_WAIT_PRIVATE, -1, nullptr, nullptr, 0); - return IsInterrupted(); - } + __atomic_fetch_sub(&Futex, 1, __ATOMIC_SEQ_CST); + while (__atomic_load_n(&Futex, __ATOMIC_ACQUIRE) == -1) + SysFutex(&Futex, FUTEX_WAIT_PRIVATE, -1, nullptr, nullptr, 0); + return IsInterrupted(); + } void Unpark() noexcept { - const int old = __atomic_fetch_add(&Futex, 1, __ATOMIC_SEQ_CST); - if (old == -1) - SysFutex(&Futex, FUTEX_WAKE_PRIVATE, -1, nullptr, nullptr, 0); - } + const int old = __atomic_fetch_add(&Futex, 1, __ATOMIC_SEQ_CST); + if (old == -1) + SysFutex(&Futex, FUTEX_WAKE_PRIVATE, -1, nullptr, nullptr, 0); + } void Interrupt() noexcept { - __atomic_store_n(&Interrupted, true, __ATOMIC_SEQ_CST); - Unpark(); - } + __atomic_store_n(&Interrupted, true, __ATOMIC_SEQ_CST); + Unpark(); + } bool IsInterrupted() const noexcept { - return __atomic_load_n(&Interrupted, __ATOMIC_ACQUIRE); - } - }; + return __atomic_load_n(&Interrupted, __ATOMIC_ACQUIRE); + } + }; #elif defined _win32_ #include <util/generic/bt_exception.h> #include <util/generic/yexception.h> namespace NActors { - class TThreadParkPad::TImpl { + class TThreadParkPad::TImpl { TAtomic Interrupted; - HANDLE EvHandle; - - public: - TImpl() - : Interrupted(false) - { - EvHandle = ::CreateEvent(0, false, false, 0); - if (!EvHandle) - ythrow TWithBackTrace<yexception>() << "::CreateEvent failed"; - } + HANDLE EvHandle; + + public: + TImpl() + : Interrupted(false) + { + EvHandle = ::CreateEvent(0, false, false, 0); + if (!EvHandle) + ythrow TWithBackTrace<yexception>() << "::CreateEvent failed"; + } ~TImpl() { - if (EvHandle) - ::CloseHandle(EvHandle); - } + if (EvHandle) + ::CloseHandle(EvHandle); + } bool Park() noexcept { - ::WaitForSingleObject(EvHandle, INFINITE); + ::WaitForSingleObject(EvHandle, INFINITE); return AtomicGet(Interrupted); - } + } void Unpark() noexcept { - ::SetEvent(EvHandle); - } + ::SetEvent(EvHandle); + } void Interrupt() noexcept { AtomicSet(Interrupted, true); - Unpark(); - } + Unpark(); + } bool IsInterrupted() const noexcept { return AtomicGet(Interrupted); - } - }; + } + }; #else #include <util/system/event.h> namespace NActors { - class TThreadParkPad::TImpl { + class TThreadParkPad::TImpl { TAtomic Interrupted; TSystemEvent Ev; - public: - TImpl() - : Interrupted(false) + public: + TImpl() + : Interrupted(false) , Ev(TSystemEvent::rAuto) - { - } + { + } ~TImpl() { - } + } bool Park() noexcept { - Ev.Wait(); + Ev.Wait(); return AtomicGet(Interrupted); - } - + } + void Unpark() noexcept { - Ev.Signal(); - } - + Ev.Signal(); + } + void Interrupt() noexcept { AtomicSet(Interrupted, true); - Unpark(); - } - + Unpark(); + } + bool IsInterrupted() const noexcept { return AtomicGet(Interrupted); - } - }; -#endif - - TThreadParkPad::TThreadParkPad() - : Impl(new TThreadParkPad::TImpl()) + } + }; +#endif + + TThreadParkPad::TThreadParkPad() + : Impl(new TThreadParkPad::TImpl()) { } - + TThreadParkPad::~TThreadParkPad() { } bool TThreadParkPad::Park() noexcept { - return Impl->Park(); + return Impl->Park(); } void TThreadParkPad::Unpark() noexcept { - Impl->Unpark(); + Impl->Unpark(); } void TThreadParkPad::Interrupt() noexcept { - Impl->Interrupt(); + Impl->Interrupt(); } bool TThreadParkPad::Interrupted() const noexcept { - return Impl->IsInterrupted(); + return Impl->IsInterrupted(); } } diff --git a/library/cpp/actors/util/unordered_cache.h b/library/cpp/actors/util/unordered_cache.h index 76f036c0cf..44ed04c01f 100644 --- a/library/cpp/actors/util/unordered_cache.h +++ b/library/cpp/actors/util/unordered_cache.h @@ -1,7 +1,7 @@ #pragma once #include "defs.h" -#include "queue_chunk.h" +#include "queue_chunk.h" template <typename T, ui32 Size = 512, ui32 ConcurrencyFactor = 1, typename TChunk = TQueueChunk<T, Size>> class TUnorderedCache : TNonCopyable { @@ -30,7 +30,7 @@ private: TReadSlot ReadSlots[Concurrency]; TWriteSlot WriteSlots[Concurrency]; - static_assert(sizeof(TChunk*) == sizeof(TAtomic), "expect sizeof(TChunk*) == sizeof(TAtomic)"); + static_assert(sizeof(TChunk*) == sizeof(TAtomic), "expect sizeof(TChunk*) == sizeof(TAtomic)"); private: struct TLockedWriter { diff --git a/library/cpp/actors/wilson/wilson_event.h b/library/cpp/actors/wilson/wilson_event.h index 7d89c33b51..e3adb092f0 100644 --- a/library/cpp/actors/wilson/wilson_event.h +++ b/library/cpp/actors/wilson/wilson_event.h @@ -132,12 +132,12 @@ namespace NWilson { return GetNodeId(*ac.ExecutorThread.ActorSystem); } - constexpr ui32 WilsonComponentId = 430; // kikimrservices: wilson + constexpr ui32 WilsonComponentId = 430; // kikimrservices: wilson template <typename TActorSystem> bool TraceEnabled(const TActorSystem& ctx) { const auto* loggerSettings = ctx.LoggerSettings(); - return loggerSettings && loggerSettings->Satisfies(NActors::NLog::PRI_DEBUG, WilsonComponentId); + return loggerSettings && loggerSettings->Satisfies(NActors::NLog::PRI_DEBUG, WilsonComponentId); } template <typename TActorSystem, typename TEvent> diff --git a/library/cpp/actors/wilson/wilson_trace.h b/library/cpp/actors/wilson/wilson_trace.h index 3d1ca50562..1e2f03a1b8 100644 --- a/library/cpp/actors/wilson/wilson_trace.h +++ b/library/cpp/actors/wilson/wilson_trace.h @@ -158,4 +158,4 @@ namespace NWilson { } }; -} +} diff --git a/library/cpp/archive/yarchive.cpp b/library/cpp/archive/yarchive.cpp index 1becc3e5da..7adefc714d 100644 --- a/library/cpp/archive/yarchive.cpp +++ b/library/cpp/archive/yarchive.cpp @@ -1,5 +1,5 @@ -#include "yarchive.h" - +#include "yarchive.h" + #include <util/generic/algorithm.h> #include <util/generic/hash.h> #include <util/generic/utility.h> @@ -14,145 +14,145 @@ #include <util/stream/zlib.h> #include <util/system/byteorder.h> #include <util/ysaveload.h> - -template <class T> + +template <class T> static inline void ESSave(IOutputStream* out, const T& t_in) { - T t = HostToLittle(t_in); - - out->Write((const void*)&t, sizeof(t)); -} - + T t = HostToLittle(t_in); + + out->Write((const void*)&t, sizeof(t)); +} + static inline void ESSave(IOutputStream* out, const TString& s) { ESSave(out, (ui32) s.size()); out->Write(s.data(), s.size()); -} - -template <class T> +} + +template <class T> static inline T ESLoad(IInputStream* in) { - T t = T(); - - if (in->Load(&t, sizeof(t)) != sizeof(t)) { + T t = T(); + + if (in->Load(&t, sizeof(t)) != sizeof(t)) { ythrow TSerializeException() << "malformed archive"; - } - - return LittleToHost(t); -} - -template <> + } + + return LittleToHost(t); +} + +template <> inline TString ESLoad<TString>(IInputStream* in) { - size_t len = ESLoad<ui32>(in); + size_t len = ESLoad<ui32>(in); TString ret; - TTempBuf tmp; - - while (len) { - const size_t toread = Min(len, tmp.Size()); - const size_t readed = in->Read(tmp.Data(), toread); - - if (!readed) { + TTempBuf tmp; + + while (len) { + const size_t toread = Min(len, tmp.Size()); + const size_t readed = in->Read(tmp.Data(), toread); + + if (!readed) { ythrow TSerializeException() << "malformed archive"; - } - - ret.append(tmp.Data(), readed); - len -= readed; - } - - return ret; -} - -namespace { - class TArchiveRecordDescriptor: public TSimpleRefCount<TArchiveRecordDescriptor> { - public: + } + + ret.append(tmp.Data(), readed); + len -= readed; + } + + return ret; +} + +namespace { + class TArchiveRecordDescriptor: public TSimpleRefCount<TArchiveRecordDescriptor> { + public: inline TArchiveRecordDescriptor(ui64 off, ui64 len, const TString& name) - : Off_(off) - , Len_(len) - , Name_(name) - { - } - + : Off_(off) + , Len_(len) + , Name_(name) + { + } + inline TArchiveRecordDescriptor(IInputStream* in) - : Off_(ESLoad<ui64>(in)) - , Len_(ESLoad<ui64>(in)) + : Off_(ESLoad<ui64>(in)) + , Len_(ESLoad<ui64>(in)) , Name_(ESLoad<TString>(in)) - { - } - + { + } + inline ~TArchiveRecordDescriptor() = default; - + inline void SaveTo(IOutputStream* out) const { - ESSave(out, Off_); - ESSave(out, Len_); - ESSave(out, Name_); - } - + ESSave(out, Off_); + ESSave(out, Len_); + ESSave(out, Name_); + } + inline const TString& Name() const noexcept { - return Name_; - } - + return Name_; + } + inline ui64 Length() const noexcept { - return Len_; - } - + return Len_; + } + inline ui64 Offset() const noexcept { - return Off_; - } - - private: - ui64 Off_; - ui64 Len_; + return Off_; + } + + private: + ui64 Off_; + ui64 Len_; TString Name_; - }; - - typedef TIntrusivePtr<TArchiveRecordDescriptor> TArchiveRecordDescriptorRef; -} - -class TArchiveWriter::TImpl { + }; + + typedef TIntrusivePtr<TArchiveRecordDescriptor> TArchiveRecordDescriptorRef; +} + +class TArchiveWriter::TImpl { using TDict = THashMap<TString, TArchiveRecordDescriptorRef>; - -public: + +public: inline TImpl(IOutputStream& out, bool compress) - : Off_(0) - , Out_(&out) - , UseCompression(compress) - { - } - + : Off_(0) + , Out_(&out) + , UseCompression(compress) + { + } + inline ~TImpl() = default; - - inline void Flush() { - Out_->Flush(); - } - - inline void Finish() { + + inline void Flush() { + Out_->Flush(); + } + + inline void Finish() { TCountingOutput out(Out_); - - { - TZLibCompress compress(&out); - - ESSave(&compress, (ui32)Dict_.size()); - + + { + TZLibCompress compress(&out); + + ESSave(&compress, (ui32)Dict_.size()); + for (const auto& kv : Dict_) { kv.second->SaveTo(&compress); - } - - ESSave(&compress, static_cast<ui8>(UseCompression)); - - compress.Finish(); - } - + } + + ESSave(&compress, static_cast<ui8>(UseCompression)); + + compress.Finish(); + } + ESSave(Out_, out.Counter()); - - Out_->Flush(); - } - + + Out_->Flush(); + } + inline void Add(const TString& key, IInputStream* src) { Y_ENSURE(!Dict_.contains(key), "key " << key.data() << " already stored"); - + TCountingOutput out(Out_); - if (UseCompression) { - TZLibCompress compress(&out); - TransferData(src, &compress); - compress.Finish(); - } else { + if (UseCompression) { + TZLibCompress compress(&out); + TransferData(src, &compress); + compress.Finish(); + } else { size_t skip_size = ArchiveWriterDefaultDataAlignment - Off_ % ArchiveWriterDefaultDataAlignment; if (skip_size == ArchiveWriterDefaultDataAlignment) { skip_size = 0; @@ -162,16 +162,16 @@ public: Off_ += 1; skip_size -= 1; } - TransferData(src, &out); - out.Finish(); - } - + TransferData(src, &out); + out.Finish(); + } + TArchiveRecordDescriptorRef descr(new TArchiveRecordDescriptor(Off_, out.Counter(), key)); - - Dict_[key] = descr; + + Dict_[key] = descr; Off_ += out.Counter(); - } - + } + inline void AddSynonym(const TString& existingKey, const TString& newKey) { Y_ENSURE(Dict_.contains(existingKey), "key " << existingKey.data() << " not stored yet"); Y_ENSURE(!Dict_.contains(newKey), "key " << newKey.data() << " already stored"); @@ -182,208 +182,208 @@ public: Dict_[newKey] = descr; } -private: - ui64 Off_; +private: + ui64 Off_; IOutputStream* Out_; - TDict Dict_; - const bool UseCompression; -}; - + TDict Dict_; + const bool UseCompression; +}; + TArchiveWriter::TArchiveWriter(IOutputStream* out, bool compress) : Impl_(new TImpl(*out, compress)) -{ -} - +{ +} + TArchiveWriter::~TArchiveWriter() { - try { - Finish(); - } catch (...) { - } -} - -void TArchiveWriter::Flush() { - if (Impl_.Get()) { - Impl_->Flush(); - } -} - -void TArchiveWriter::Finish() { - if (Impl_.Get()) { - Impl_->Finish(); - Impl_.Destroy(); - } -} - + try { + Finish(); + } catch (...) { + } +} + +void TArchiveWriter::Flush() { + if (Impl_.Get()) { + Impl_->Flush(); + } +} + +void TArchiveWriter::Finish() { + if (Impl_.Get()) { + Impl_->Finish(); + Impl_.Destroy(); + } +} + void TArchiveWriter::Add(const TString& key, IInputStream* src) { Y_ENSURE(Impl_.Get(), "archive already closed"); - - Impl_->Add(key, src); -} - + + Impl_->Add(key, src); +} + void TArchiveWriter::AddSynonym(const TString& existingKey, const TString& newKey) { Y_ENSURE(Impl_.Get(), "archive already closed"); Impl_->AddSynonym(existingKey, newKey); } -namespace { - class TArchiveInputStreamBase { - public: - inline TArchiveInputStreamBase(const TBlob& b) - : Blob_(b) - , Input_(b.Data(), b.Size()) - { - } - - protected: - TBlob Blob_; - TMemoryInput Input_; - }; - - class TArchiveInputStream: public TArchiveInputStreamBase, public TZLibDecompress { - public: - inline TArchiveInputStream(const TBlob& b) - : TArchiveInputStreamBase(b) +namespace { + class TArchiveInputStreamBase { + public: + inline TArchiveInputStreamBase(const TBlob& b) + : Blob_(b) + , Input_(b.Data(), b.Size()) + { + } + + protected: + TBlob Blob_; + TMemoryInput Input_; + }; + + class TArchiveInputStream: public TArchiveInputStreamBase, public TZLibDecompress { + public: + inline TArchiveInputStream(const TBlob& b) + : TArchiveInputStreamBase(b) , TZLibDecompress(&Input_) - { - } - + { + } + ~TArchiveInputStream() override = default; - }; -} - -class TArchiveReader::TImpl { + }; +} + +class TArchiveReader::TImpl { typedef THashMap<TString, TArchiveRecordDescriptorRef> TDict; - -public: - inline TImpl(const TBlob& blob) - : Blob_(blob) - , UseDecompression(true) - { - ReadDict(); - } - + +public: + inline TImpl(const TBlob& blob) + : Blob_(blob) + , UseDecompression(true) + { + ReadDict(); + } + inline ~TImpl() = default; - - inline void ReadDict() { + + inline void ReadDict() { Y_ENSURE(Blob_.Size() >= sizeof(ui64), "too small blob"); - - const char* end = (const char*)Blob_.End(); - const char* ptr = end - sizeof(ui64); - ui64 dictlen = 0; - memcpy(&dictlen, ptr, sizeof(ui64)); - dictlen = LittleToHost(dictlen); - + + const char* end = (const char*)Blob_.End(); + const char* ptr = end - sizeof(ui64); + ui64 dictlen = 0; + memcpy(&dictlen, ptr, sizeof(ui64)); + dictlen = LittleToHost(dictlen); + Y_ENSURE(dictlen <= Blob_.Size() - sizeof(ui64), "bad blob"); - - const char* beg = ptr - dictlen; - TMemoryInput mi(beg, dictlen); + + const char* beg = ptr - dictlen; + TMemoryInput mi(beg, dictlen); TZLibDecompress d(&mi); - const ui32 count = ESLoad<ui32>(&d); - - for (size_t i = 0; i < count; ++i) { - TArchiveRecordDescriptorRef descr(new TArchiveRecordDescriptor(&d)); - - Recs_.push_back(descr); - Dict_[descr->Name()] = descr; - } + const ui32 count = ESLoad<ui32>(&d); + + for (size_t i = 0; i < count; ++i) { + TArchiveRecordDescriptorRef descr(new TArchiveRecordDescriptor(&d)); + + Recs_.push_back(descr); + Dict_[descr->Name()] = descr; + } Sort(Recs_.begin(), Recs_.end(), [](const auto& lhs, const auto& rhs) -> bool { return lhs->Offset() < rhs->Offset(); }); - - try { - UseDecompression = static_cast<bool>(ESLoad<ui8>(&d)); - } catch (const TSerializeException&) { - // that's ok - just old format - UseDecompression = true; - } - } - + + try { + UseDecompression = static_cast<bool>(ESLoad<ui8>(&d)); + } catch (const TSerializeException&) { + // that's ok - just old format + UseDecompression = true; + } + } + inline size_t Count() const noexcept { - return Recs_.size(); - } - + return Recs_.size(); + } + inline TString KeyByIndex(size_t n) const { - if (n < Count()) { - return Recs_[n]->Name(); - } - - ythrow yexception() << "incorrect index"; - } + if (n < Count()) { + return Recs_[n]->Name(); + } + + ythrow yexception() << "incorrect index"; + } inline bool Has(const TStringBuf key) const { return Dict_.contains(key); - } + } inline TAutoPtr<IInputStream> ObjectByKey(const TStringBuf key) const { - TBlob subBlob = BlobByKey(key); - - if (UseDecompression) { - return new TArchiveInputStream(subBlob); - } else { - return new TMemoryInput(subBlob.Data(), subBlob.Length()); + TBlob subBlob = BlobByKey(key); + + if (UseDecompression) { + return new TArchiveInputStream(subBlob); + } else { + return new TMemoryInput(subBlob.Data(), subBlob.Length()); } - } + } inline TBlob ObjectBlobByKey(const TStringBuf key) const { - TBlob subBlob = BlobByKey(key); + TBlob subBlob = BlobByKey(key); - if (UseDecompression) { - TArchiveInputStream st(subBlob); - return TBlob::FromStream(st); - } else { - return subBlob; + if (UseDecompression) { + TArchiveInputStream st(subBlob); + return TBlob::FromStream(st); + } else { + return subBlob; } - } + } inline TBlob BlobByKey(const TStringBuf key) const { const auto it = Dict_.find(key); - + Y_ENSURE(it != Dict_.end(), "key " << key.data() << " not found"); - - const size_t off = it->second->Offset(); - const size_t len = it->second->Length(); - - /* - * TODO - overflow check - */ - - return Blob_.SubBlob(off, off + len); - } - - inline bool Compressed() const { - return UseDecompression; - } - -private: - TBlob Blob_; + + const size_t off = it->second->Offset(); + const size_t len = it->second->Length(); + + /* + * TODO - overflow check + */ + + return Blob_.SubBlob(off, off + len); + } + + inline bool Compressed() const { + return UseDecompression; + } + +private: + TBlob Blob_; TVector<TArchiveRecordDescriptorRef> Recs_; - TDict Dict_; - bool UseDecompression; -}; - -TArchiveReader::TArchiveReader(const TBlob& data) - : Impl_(new TImpl(data)) -{ -} - + TDict Dict_; + bool UseDecompression; +}; + +TArchiveReader::TArchiveReader(const TBlob& data) + : Impl_(new TImpl(data)) +{ +} + TArchiveReader::~TArchiveReader() {} - + size_t TArchiveReader::Count() const noexcept { - return Impl_->Count(); -} - + return Impl_->Count(); +} + TString TArchiveReader::KeyByIndex(size_t n) const { - return Impl_->KeyByIndex(n); -} - + return Impl_->KeyByIndex(n); +} + bool TArchiveReader::Has(const TStringBuf key) const { return Impl_->Has(key); } TAutoPtr<IInputStream> TArchiveReader::ObjectByKey(const TStringBuf key) const { - return Impl_->ObjectByKey(key); -} + return Impl_->ObjectByKey(key); +} TBlob TArchiveReader::ObjectBlobByKey(const TStringBuf key) const { return Impl_->ObjectBlobByKey(key); diff --git a/library/cpp/archive/yarchive.h b/library/cpp/archive/yarchive.h index 8120bcb940..a4e468ded2 100644 --- a/library/cpp/archive/yarchive.h +++ b/library/cpp/archive/yarchive.h @@ -1,10 +1,10 @@ #pragma once - + #include "models_archive_reader.h" #include <util/generic/fwd.h> #include <util/generic/ptr.h> - + class IInputStream; class IOutputStream; @@ -14,26 +14,26 @@ class TBlob; //noncompressed data will be stored with default alignment DEVTOOLS-4384 static constexpr size_t ArchiveWriterDefaultDataAlignment = 16; -class TArchiveWriter { -public: +class TArchiveWriter { +public: explicit TArchiveWriter(IOutputStream* out, bool compress = true); ~TArchiveWriter(); - - void Flush(); - void Finish(); + + void Flush(); + void Finish(); void Add(const TString& key, IInputStream* src); void AddSynonym(const TString& existingKey, const TString& newKey); - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + class TArchiveReader : public IModelsArchiveReader { -public: +public: explicit TArchiveReader(const TBlob& data); ~TArchiveReader() override; - + size_t Count() const noexcept override; TString KeyByIndex(size_t n) const override; bool Has(TStringBuf key) const override; @@ -41,8 +41,8 @@ public: TBlob ObjectBlobByKey(TStringBuf key) const override; TBlob BlobByKey(TStringBuf key) const override; bool Compressed() const override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; diff --git a/library/cpp/archive/yarchive_ut.cpp b/library/cpp/archive/yarchive_ut.cpp index 602a1cdbbd..d4abdcd44c 100644 --- a/library/cpp/archive/yarchive_ut.cpp +++ b/library/cpp/archive/yarchive_ut.cpp @@ -1,67 +1,67 @@ -#include "yarchive.h" - +#include "yarchive.h" + #include <library/cpp/testing/unittest/registar.h> - -#include <util/string/cast.h> + +#include <util/string/cast.h> #include <util/stream/file.h> -#include <util/system/tempfile.h> +#include <util/system/tempfile.h> #include <util/memory/blob.h> - -class TArchiveTest: public TTestBase { - UNIT_TEST_SUITE(TArchiveTest) - UNIT_TEST(TestCreate); - UNIT_TEST(TestRead); + +class TArchiveTest: public TTestBase { + UNIT_TEST_SUITE(TArchiveTest) + UNIT_TEST(TestCreate); + UNIT_TEST(TestRead); UNIT_TEST(TestOffsetOrder); - UNIT_TEST_SUITE_END(); - -private: + UNIT_TEST_SUITE_END(); + +private: void CreateArchive(); - void TestCreate(); - void TestRead(); + void TestCreate(); + void TestRead(); void TestOffsetOrder(); -}; - -UNIT_TEST_SUITE_REGISTRATION(TArchiveTest); - -#define ARCHIVE "./test.ar" - +}; + +UNIT_TEST_SUITE_REGISTRATION(TArchiveTest); + +#define ARCHIVE "./test.ar" + void TArchiveTest::CreateArchive() { TFixedBufferFileOutput out(ARCHIVE); - TArchiveWriter w(&out); - - for (size_t i = 0; i < 1000; ++i) { + TArchiveWriter w(&out); + + for (size_t i = 0; i < 1000; ++i) { const TString path = "/" + ToString(i); const TString data = "data" + ToString(i * 1000) + "dataend"; - TStringInput si(data); - - w.Add(path, &si); - } - - w.Finish(); - out.Finish(); -} - + TStringInput si(data); + + w.Add(path, &si); + } + + w.Finish(); + out.Finish(); +} + void TArchiveTest::TestCreate() { CreateArchive(); TTempFile tmpFile(ARCHIVE); } -void TArchiveTest::TestRead() { +void TArchiveTest::TestRead() { CreateArchive(); - TTempFile tmpFile(ARCHIVE); + TTempFile tmpFile(ARCHIVE); TBlob blob = TBlob::FromFileSingleThreaded(ARCHIVE); - TArchiveReader r(blob); - - UNIT_ASSERT_EQUAL(r.Count(), 1000); - - for (size_t i = 0; i < 1000; ++i) { + TArchiveReader r(blob); + + UNIT_ASSERT_EQUAL(r.Count(), 1000); + + for (size_t i = 0; i < 1000; ++i) { const TString key = "/" + ToString(i); TAutoPtr<IInputStream> is = r.ObjectByKey(key); const TString data = is->ReadAll(); - - UNIT_ASSERT_EQUAL(data, "data" + ToString(i * 1000) + "dataend"); - } -} + + UNIT_ASSERT_EQUAL(data, "data" + ToString(i * 1000) + "dataend"); + } +} void TArchiveTest::TestOffsetOrder() { CreateArchive(); diff --git a/library/cpp/balloc/balloc.cpp b/library/cpp/balloc/balloc.cpp index fab489db4c..a81f73c65a 100644 --- a/library/cpp/balloc/balloc.cpp +++ b/library/cpp/balloc/balloc.cpp @@ -3,40 +3,40 @@ namespace NBalloc { - static constexpr size_t ALIVE_SIGNATURE = 0xaULL << 56; - static constexpr size_t DISABLED_SIGNATURE = 0xbULL << 56; - static constexpr size_t SIGNATURE_MASK = 0xfULL << 56; + static constexpr size_t ALIVE_SIGNATURE = 0xaULL << 56; + static constexpr size_t DISABLED_SIGNATURE = 0xbULL << 56; + static constexpr size_t SIGNATURE_MASK = 0xfULL << 56; static constexpr size_t MINIMAL_ALIGNMENT = sizeof(NBalloc::TAllocHeader); static_assert(((MINIMAL_ALIGNMENT - 1) & MINIMAL_ALIGNMENT) == 0, "invalid BALLOC_MINIMAL_ALIGNMENT"); - static Y_FORCE_INLINE void* Malloc(size_t size) { - TLS& ltls = tls; + static Y_FORCE_INLINE void* Malloc(size_t size) { + TLS& ltls = tls; size = Align(size, sizeof(TAllocHeader)); - if (Y_UNLIKELY(ltls.Mode == Empty || ltls.Mode == ToBeEnabled)) { + if (Y_UNLIKELY(ltls.Mode == Empty || ltls.Mode == ToBeEnabled)) { Init(ltls); - } - if (Y_LIKELY(ltls.Mode != Disabled)) { + } + if (Y_LIKELY(ltls.Mode != Disabled)) { TAllocHeader* allocHeader = AllocateRaw(size, ALIVE_SIGNATURE); return allocHeader + 1; - } else { - // ltls.Mode == Disabled - const size_t extsize = size + sizeof(TAllocHeader); + } else { + // ltls.Mode == Disabled + const size_t extsize = size + sizeof(TAllocHeader); TAllocHeader* allocHeader = (TAllocHeader*)LibcMalloc(extsize); allocHeader->Encode(allocHeader, size, DISABLED_SIGNATURE); return allocHeader + 1; } - } + } - static void Y_FORCE_INLINE Free(void* ptr) { + static void Y_FORCE_INLINE Free(void* ptr) { if (ptr == nullptr) { - return; + return; } TAllocHeader* allocHeader = ((TAllocHeader*)ptr) - 1; size_t size = allocHeader->AllocSize; - const size_t signature = size & SIGNATURE_MASK; - if (Y_LIKELY(signature == ALIVE_SIGNATURE)) { + const size_t signature = size & SIGNATURE_MASK; + if (Y_LIKELY(signature == ALIVE_SIGNATURE)) { allocHeader->AllocSize = 0; // abort later on double free #ifdef DBG_FILL_MEMORY memset(ptr, 0xde, size - signature); @@ -45,62 +45,62 @@ namespace NBalloc { if (NAllocStats::IsEnabled()) { NAllocStats::DecThreadAllocStats(size - signature); } - } else if (signature == DISABLED_SIGNATURE) { + } else if (signature == DISABLED_SIGNATURE) { LibcFree(allocHeader->Block); - } else { - NMalloc::AbortFromCorruptedAllocator(); - } + } else { + NMalloc::AbortFromCorruptedAllocator(); + } } - static bool Y_FORCE_INLINE IsOwnedByBalloc(void* ptr) { - TAllocHeader* allocHeader = ((TAllocHeader*)ptr) - 1; - size_t size = allocHeader->AllocSize; - const size_t signature = size & SIGNATURE_MASK; - if (signature == ALIVE_SIGNATURE) { - return true; - } else if (signature == DISABLED_SIGNATURE) { - return false; - } + static bool Y_FORCE_INLINE IsOwnedByBalloc(void* ptr) { + TAllocHeader* allocHeader = ((TAllocHeader*)ptr) - 1; + size_t size = allocHeader->AllocSize; + const size_t signature = size & SIGNATURE_MASK; + if (signature == ALIVE_SIGNATURE) { + return true; + } else if (signature == DISABLED_SIGNATURE) { + return false; + } NMalloc::AbortFromCorruptedAllocator(); - Y_UNREACHABLE(); + Y_UNREACHABLE(); } - static void Y_FORCE_INLINE Disable() { + static void Y_FORCE_INLINE Disable() { #if defined(_musl_) - // just skip it + // just skip it #else - tls.Mode = Disabled; + tls.Mode = Disabled; #endif - } + } - static void Y_FORCE_INLINE Enable() { + static void Y_FORCE_INLINE Enable() { tls.Mode = ToBeEnabled; - } + } - static bool Y_FORCE_INLINE IsDisabled() { - return tls.Mode == Disabled; - } + static bool Y_FORCE_INLINE IsDisabled() { + return tls.Mode == Disabled; + } }; -#if defined(Y_COVER_PTR) -void* CoverPtr(void* ptr, size_t len) noexcept; -void* UncoverPtr(void* ptr) noexcept; -#endif - -extern "C" void* malloc(size_t size) { -#if defined(Y_COVER_PTR) - return CoverPtr(NBalloc::Malloc(size + 32), size); -#else +#if defined(Y_COVER_PTR) +void* CoverPtr(void* ptr, size_t len) noexcept; +void* UncoverPtr(void* ptr) noexcept; +#endif + +extern "C" void* malloc(size_t size) { +#if defined(Y_COVER_PTR) + return CoverPtr(NBalloc::Malloc(size + 32), size); +#else return NBalloc::Malloc(size); -#endif +#endif } -extern "C" void free(void* data) { -#if defined(Y_COVER_PTR) - NBalloc::Free(UncoverPtr(data)); -#else +extern "C" void free(void* data) { +#if defined(Y_COVER_PTR) + NBalloc::Free(UncoverPtr(data)); +#else NBalloc::Free(data); -#endif +#endif } #if defined(USE_INTELCC) || defined(_darwin_) || defined(_freebsd_) || defined(_STLPORT_VERSION) @@ -110,19 +110,19 @@ extern "C" void free(void* data) { #endif void* operator new(size_t size) { -#if defined(Y_COVER_PTR) - return malloc(size); -#else +#if defined(Y_COVER_PTR) + return malloc(size); +#else return NBalloc::Malloc(size); -#endif +#endif } int posix_memalign(void** memptr, const size_t alignment, const size_t size) { -#if defined(Y_COVER_PTR) - (void)alignment; - *memptr = malloc(size); - return 0; -#else +#if defined(Y_COVER_PTR) + (void)alignment; + *memptr = malloc(size); + return 0; +#else if (((alignment - 1) & alignment) != 0 || alignment < sizeof(void*)) { return EINVAL; } @@ -141,98 +141,98 @@ int posix_memalign(void** memptr, const size_t alignment, const size_t size) { } *memptr = alignedPtr; return 0; -#endif +#endif } void* operator new(size_t size, const std::nothrow_t&) OP_THROWNOTHING { -#if defined(Y_COVER_PTR) - return malloc(size); -#else +#if defined(Y_COVER_PTR) + return malloc(size); +#else return NBalloc::Malloc(size); -#endif +#endif } void operator delete(void* p)OP_THROWNOTHING { -#if defined(Y_COVER_PTR) - free(p); -#else +#if defined(Y_COVER_PTR) + free(p); +#else NBalloc::Free(p); -#endif +#endif } void operator delete(void* p, const std::nothrow_t&)OP_THROWNOTHING { -#if defined(Y_COVER_PTR) - free(p); -#else +#if defined(Y_COVER_PTR) + free(p); +#else NBalloc::Free(p); -#endif +#endif } void* operator new[](size_t size) { -#if defined(Y_COVER_PTR) - return malloc(size); -#else +#if defined(Y_COVER_PTR) + return malloc(size); +#else return NBalloc::Malloc(size); -#endif +#endif } void* operator new[](size_t size, const std::nothrow_t&) OP_THROWNOTHING { -#if defined(Y_COVER_PTR) - return malloc(size); -#else +#if defined(Y_COVER_PTR) + return malloc(size); +#else return NBalloc::Malloc(size); -#endif +#endif } void operator delete[](void* p) OP_THROWNOTHING { -#if defined(Y_COVER_PTR) - free(p); -#else +#if defined(Y_COVER_PTR) + free(p); +#else NBalloc::Free(p); -#endif +#endif } void operator delete[](void* p, const std::nothrow_t&) OP_THROWNOTHING { -#if defined(Y_COVER_PTR) - free(p); -#else +#if defined(Y_COVER_PTR) + free(p); +#else NBalloc::Free(p); -#endif +#endif } extern "C" void* calloc(size_t n, size_t elemSize) { const size_t size = n * elemSize; - + if (elemSize != 0 && size / elemSize != n) { return nullptr; } - -#if defined(Y_COVER_PTR) - void* result = malloc(size); -#else + +#if defined(Y_COVER_PTR) + void* result = malloc(size); +#else void* result = NBalloc::Malloc(size); -#endif - +#endif + if (result) { memset(result, 0, size); } - + return result; } extern "C" void cfree(void* ptr) { -#if defined(Y_COVER_PTR) - free(ptr); -#else +#if defined(Y_COVER_PTR) + free(ptr); +#else NBalloc::Free(ptr); -#endif +#endif } -#if defined(Y_COVER_PTR) -static inline void* DoRealloc(void* oldPtr, size_t newSize) { -#else +#if defined(Y_COVER_PTR) +static inline void* DoRealloc(void* oldPtr, size_t newSize) { +#else extern "C" void* realloc(void* oldPtr, size_t newSize) { -#endif +#endif if (!oldPtr) { void* result = NBalloc::Malloc(newSize); return result; @@ -241,11 +241,11 @@ extern "C" void* realloc(void* oldPtr, size_t newSize) { NBalloc::Free(oldPtr); return nullptr; } - void* newPtr = NBalloc::Malloc(newSize); + void* newPtr = NBalloc::Malloc(newSize); if (!newPtr) { return nullptr; } - NBalloc::TAllocHeader* header = (NBalloc::TAllocHeader*)oldPtr - 1; + NBalloc::TAllocHeader* header = (NBalloc::TAllocHeader*)oldPtr - 1; const size_t oldSize = header->AllocSize & ~NBalloc::SIGNATURE_MASK; const size_t signature = header->AllocSize & NBalloc::SIGNATURE_MASK; if (Y_LIKELY((signature == NBalloc::ALIVE_SIGNATURE) || (signature == NBalloc::DISABLED_SIGNATURE))) { @@ -257,24 +257,24 @@ extern "C" void* realloc(void* oldPtr, size_t newSize) { return nullptr; } -#if defined(Y_COVER_PTR) -extern "C" void* realloc(void* oldPtr, size_t newSize) { - if (!oldPtr) { - return malloc(newSize); - } - - if (!newSize) { - free(oldPtr); - - return nullptr; - } - - return CoverPtr(DoRealloc(UncoverPtr(oldPtr), newSize + 32), newSize); -} -#endif - +#if defined(Y_COVER_PTR) +extern "C" void* realloc(void* oldPtr, size_t newSize) { + if (!oldPtr) { + return malloc(newSize); + } + + if (!newSize) { + free(oldPtr); + + return nullptr; + } + + return CoverPtr(DoRealloc(UncoverPtr(oldPtr), newSize + 32), newSize); +} +#endif + // Only for testing purposes. Never use in production. -extern "C" bool IsOwnedByBalloc(void* ptr) { +extern "C" bool IsOwnedByBalloc(void* ptr) { return NBalloc::IsOwnedByBalloc(ptr); } @@ -300,7 +300,7 @@ extern "C" void* valloc(size_t size) { return memalign(NBalloc::PAGE_ELEM, size); } -#if !defined(_MSC_VER) && !defined(_freebsd_) +#if !defined(_MSC_VER) && !defined(_freebsd_) // Workaround for pthread_create bug in linux. extern "C" void* __libc_memalign(size_t alignment, size_t size) { return memalign(alignment, size); diff --git a/library/cpp/balloc/malloc-info.cpp b/library/cpp/balloc/malloc-info.cpp index 604b1fb145..008606e0a8 100644 --- a/library/cpp/balloc/malloc-info.cpp +++ b/library/cpp/balloc/malloc-info.cpp @@ -1,26 +1,26 @@ #include <library/cpp/malloc/api/malloc.h> -#include <string.h> +#include <string.h> using namespace NMalloc; -extern "C" void DisableBalloc(); +extern "C" void DisableBalloc(); extern "C" void EnableBalloc(); extern "C" bool BallocDisabled(); -namespace { +namespace { bool SetAllocParam(const char* name, const char* value) { - if (strcmp(name, "disable") == 0) { + if (strcmp(name, "disable") == 0) { if (value == nullptr || strcmp(value, "false") != 0) { // all values other than "false" are considred to be "true" for compatibility DisableBalloc(); } else { EnableBalloc(); } - return true; - } - return false; - } + return true; + } + return false; + } bool CheckAllocParam(const char* name, bool defaultValue) { if (strcmp(name, "disable") == 0) { @@ -28,14 +28,14 @@ namespace { } return defaultValue; } -} - +} + TMallocInfo NMalloc::MallocInfo() { TMallocInfo r; - + r.Name = "balloc"; - r.SetParam = SetAllocParam; + r.SetParam = SetAllocParam; r.CheckParam = CheckAllocParam; - + return r; } diff --git a/library/cpp/binsaver/bin_saver.cpp b/library/cpp/binsaver/bin_saver.cpp index fe0775af9f..36ec9efdfc 100644 --- a/library/cpp/binsaver/bin_saver.cpp +++ b/library/cpp/binsaver/bin_saver.cpp @@ -1,19 +1,19 @@ #include "bin_saver.h" -TClassFactory<IObjectBase>* pSaverClasses; +TClassFactory<IObjectBase>* pSaverClasses; void StartRegisterSaveload() { - if (!pSaverClasses) + if (!pSaverClasses) pSaverClasses = new TClassFactory<IObjectBase>; } struct SBasicChunkInit { - ~SBasicChunkInit() { - if (pSaverClasses) - delete pSaverClasses; - } + ~SBasicChunkInit() { + if (pSaverClasses) + delete pSaverClasses; + } } initSaver; ////////////////////////////////////////////////////////////////////////// -void IBinSaver::StoreObject(IObjectBase* pObject) { +void IBinSaver::StoreObject(IObjectBase* pObject) { if (pObject) { Y_ASSERT(pSaverClasses->GetObjectTypeID(pObject) != -1 && "trying to save unregistered object"); } @@ -50,7 +50,7 @@ void IBinSaver::StoreObject(IObjectBase* pObject) { } } -IObjectBase* IBinSaver::LoadObject() { +IObjectBase* IBinSaver::LoadObject() { ui64 ptrId = 0; DataChunk(&ptrId, sizeof(ptrId)); if (ptrId != 0) { @@ -61,7 +61,7 @@ IObjectBase* IBinSaver::LoadObject() { return pFound->second; int typeId; DataChunk(&typeId, sizeof(typeId)); - IObjectBase* pObj = pSaverClasses->CreateObject(typeId); + IObjectBase* pObj = pSaverClasses->CreateObject(typeId); Y_ASSERT(pObj != nullptr); if (pObj == nullptr) { fprintf(stderr, "IBinSaver: trying to load unregistered object\n"); diff --git a/library/cpp/binsaver/bin_saver.h b/library/cpp/binsaver/bin_saver.h index 412424889f..c8b1873938 100644 --- a/library/cpp/binsaver/bin_saver.h +++ b/library/cpp/binsaver/bin_saver.h @@ -2,9 +2,9 @@ #include "buffered_io.h" #include "class_factory.h" - + #include <library/cpp/containers/2d_array/2d_array.h> - + #include <util/generic/hash_set.h> #include <util/generic/buffer.h> #include <util/generic/list.h> @@ -17,16 +17,16 @@ #include <array> #include <bitset> -#include <list> -#include <string> +#include <list> +#include <string> #ifdef _MSC_VER -#pragma warning(disable : 4127) +#pragma warning(disable : 4127) #endif -enum ESaverMode { - SAVER_MODE_READ = 1, - SAVER_MODE_WRITE = 2, +enum ESaverMode { + SAVER_MODE_READ = 1, + SAVER_MODE_WRITE = 2, SAVER_MODE_WRITE_COMPRESSED = 3, }; @@ -43,7 +43,7 @@ namespace NBinSaverInternals { } ////////////////////////////////////////////////////////////////////////// -struct IBinSaver { +struct IBinSaver { public: typedef unsigned char chunk_id; typedef ui32 TStoredSize; // changing this will break compatibility @@ -62,18 +62,18 @@ private: // return 0; // } // }; - template <class T, typename = decltype(std::declval<T*>()->T::operator&(std::declval<IBinSaver&>()))> + template <class T, typename = decltype(std::declval<T*>()->T::operator&(std::declval<IBinSaver&>()))> void CallObjectSerialize(T* p, NBinSaverInternals::TOverloadPriority<2>) { // highest priority - will be resolved first if enabled - // Note: p->operator &(*this) would lead to infinite recursion - p->T::operator&(*this); + // Note: p->operator &(*this) would lead to infinite recursion + p->T::operator&(*this); } - template <class T, typename = decltype(std::declval<T&>() & std::declval<IBinSaver&>())> + template <class T, typename = decltype(std::declval<T&>() & std::declval<IBinSaver&>())> void CallObjectSerialize(T* p, NBinSaverInternals::TOverloadPriority<1>) { // lower priority - will be resolved second if enabled (*p) & (*this); } - template <class T> + template <class T> void CallObjectSerialize(T* p, NBinSaverInternals::TOverloadPriority<0>) { // lower priority - will be resolved last #if (!defined(_MSC_VER)) // In MSVC __has_trivial_copy returns false to enums, primitive types and arrays. @@ -83,8 +83,8 @@ private: } // vector - template <class T, class TA> - void DoVector(TVector<T, TA>& data) { + template <class T, class TA> + void DoVector(TVector<T, TA>& data) { TStoredSize nSize; if (IsReading()) { data.clear(); @@ -99,23 +99,23 @@ private: Add(1, &data[i]); } - template <class T, int N> - void DoArray(T (&data)[N]) { + template <class T, int N> + void DoArray(T (&data)[N]) { for (size_t i = 0; i < N; i++) { Add(1, &(data[i])); } } - template <typename TLarge> - void CheckOverflow(TStoredSize nSize, TLarge origSize) { + template <typename TLarge> + void CheckOverflow(TStoredSize nSize, TLarge origSize) { if (nSize != origSize) { fprintf(stderr, "IBinSaver: object size is too large to be serialized (%" PRIu32 " != %" PRIu64 ")\n", nSize, (ui64)origSize); abort(); } } - template <class T, class TA> - void DoDataVector(TVector<T, TA>& data) { + template <class T, class TA> + void DoDataVector(TVector<T, TA>& data) { TStoredSize nSize = data.size(); CheckOverflow(nSize, data.size()); Add(1, &nSize); @@ -127,8 +127,8 @@ private: DataChunk(&data[0], sizeof(T) * nSize); } - template <class AM> - void DoAnyMap(AM& data) { + template <class AM> + void DoAnyMap(AM& data) { if (IsReading()) { data.clear(); TStoredSize nSize; @@ -138,7 +138,7 @@ private: for (TStoredSize i = 0; i < nSize; ++i) Add(1, &indices[i]); for (TStoredSize i = 0; i < nSize; ++i) - Add(2, &data[indices[i]]); + Add(2, &data[indices[i]]); } else { TStoredSize nSize = data.size(); CheckOverflow(nSize, data.size()); @@ -148,17 +148,17 @@ private: indices.resize(nSize); TStoredSize i = 1; for (auto pos = data.begin(); pos != data.end(); ++pos, ++i) - indices[nSize - i] = pos->first; + indices[nSize - i] = pos->first; for (TStoredSize j = 0; j < nSize; ++j) Add(1, &indices[j]); for (TStoredSize j = 0; j < nSize; ++j) - Add(2, &data[indices[j]]); + Add(2, &data[indices[j]]); } } // hash_multimap - template <class AMM> - void DoAnyMultiMap(AMM& data) { + template <class AMM> + void DoAnyMultiMap(AMM& data) { if (IsReading()) { data.clear(); TStoredSize nSize; @@ -178,14 +178,14 @@ private: CheckOverflow(nSize, data.size()); Add(3, &nSize); for (auto pos = data.begin(); pos != data.end(); ++pos) - Add(1, (typename AMM::key_type*)(&pos->first)); + Add(1, (typename AMM::key_type*)(&pos->first)); for (auto pos = data.begin(); pos != data.end(); ++pos) Add(2, &pos->second); } } - template <class T> - void DoAnySet(T& data) { + template <class T> + void DoAnySet(T& data) { if (IsReading()) { data.clear(); TStoredSize nSize; @@ -207,18 +207,18 @@ private: } // 2D array - template <class T> - void Do2DArray(TArray2D<T>& a) { + template <class T> + void Do2DArray(TArray2D<T>& a) { int nXSize = a.GetXSize(), nYSize = a.GetYSize(); Add(1, &nXSize); Add(2, &nYSize); if (IsReading()) a.SetSizes(nXSize, nYSize); for (int i = 0; i < nXSize * nYSize; i++) - Add(3, &a[i / nXSize][i % nXSize]); + Add(3, &a[i / nXSize][i % nXSize]); } - template <class T> - void Do2DArrayData(TArray2D<T>& a) { + template <class T> + void Do2DArrayData(TArray2D<T>& a) { int nXSize = a.GetXSize(), nYSize = a.GetYSize(); Add(1, &nXSize); Add(2, &nYSize); @@ -228,8 +228,8 @@ private: DataChunk(&a[0][0], sizeof(T) * nXSize * nYSize); } // strings - template <class TStringType> - void DataChunkStr(TStringType& data, i64 elemSize) { + template <class TStringType> + void DataChunkStr(TStringType& data, i64 elemSize) { if (bRead) { TStoredSize nCount = 0; File.Read(&nCount, sizeof(TStoredSize)); @@ -243,20 +243,20 @@ private: File.Write(data.c_str(), nCount * elemSize); } } - void DataChunkString(std::string& data) { + void DataChunkString(std::string& data) { DataChunkStr(data, sizeof(char)); } - void DataChunkStroka(TString& data) { + void DataChunkStroka(TString& data) { DataChunkStr(data, sizeof(TString::char_type)); } - void DataChunkWtroka(TUtf16String& data) { + void DataChunkWtroka(TUtf16String& data) { DataChunkStr(data, sizeof(wchar16)); } - void DataChunk(void* pData, i64 nSize) { + void DataChunk(void* pData, i64 nSize) { i64 chunkSize = 1 << 30; for (i64 offset = 0; offset < nSize; offset += chunkSize) { - void* ptr = (char*)pData + offset; + void* ptr = (char*)pData + offset; i64 size = offset + chunkSize < nSize ? chunkSize : (nSize - offset); if (bRead) File.Read(ptr, size); @@ -266,7 +266,7 @@ private: } // storing/loading pointers to objects - void StoreObject(IObjectBase* pObject); + void StoreObject(IObjectBase* pObject); IObjectBase* LoadObject(); bool bRead; @@ -278,38 +278,38 @@ private: typedef THashMap<void*, ui32> PtrIdHash; TAutoPtr<PtrIdHash> PtrIds; - typedef THashMap<ui64, TPtr<IObjectBase>> CObjectsHash; + typedef THashMap<ui64, TPtr<IObjectBase>> CObjectsHash; TAutoPtr<CObjectsHash> Objects; TVector<IObjectBase*> ObjectQueue; - + public: - bool IsReading() { - return bRead; - } - void AddRawData(const chunk_id, void* pData, i64 nSize) { - DataChunk(pData, nSize); - } + bool IsReading() { + return bRead; + } + void AddRawData(const chunk_id, void* pData, i64 nSize) { + DataChunk(pData, nSize); + } // return type of Add() is used to detect specialized serializer (see HasNonTrivialSerializer below) - template <class T> - char Add(const chunk_id, T* p) { + template <class T> + char Add(const chunk_id, T* p) { CallObjectSerialize(p, NBinSaverInternals::TOverloadPriority<2>()); return 0; } - int Add(const chunk_id, std::string* pStr) { + int Add(const chunk_id, std::string* pStr) { DataChunkString(*pStr); return 0; } - int Add(const chunk_id, TString* pStr) { + int Add(const chunk_id, TString* pStr) { DataChunkStroka(*pStr); return 0; } - int Add(const chunk_id, TUtf16String* pStr) { + int Add(const chunk_id, TUtf16String* pStr) { DataChunkWtroka(*pStr); return 0; } - int Add(const chunk_id, TBlob* blob) { + int Add(const chunk_id, TBlob* blob) { if (bRead) { ui64 size = 0; File.Read(&size, sizeof(size)); @@ -325,8 +325,8 @@ public: } return 0; } - template <class T1, class TA> - int Add(const chunk_id, TVector<T1, TA>* pVec) { + template <class T1, class TA> + int Add(const chunk_id, TVector<T1, TA>* pVec) { if (HasNonTrivialSerializer<T1>(0u)) DoVector(*pVec); else @@ -334,8 +334,8 @@ public: return 0; } - template <class T, int N> - int Add(const chunk_id, T (*pVec)[N]) { + template <class T, int N> + int Add(const chunk_id, T (*pVec)[N]) { if (HasNonTrivialSerializer<T>(0u)) DoArray(*pVec); else @@ -343,49 +343,49 @@ public: return 0; } - template <class T1, class T2, class T3, class T4> - int Add(const chunk_id, TMap<T1, T2, T3, T4>* pMap) { + template <class T1, class T2, class T3, class T4> + int Add(const chunk_id, TMap<T1, T2, T3, T4>* pMap) { DoAnyMap(*pMap); return 0; } - template <class T1, class T2, class T3, class T4, class T5> - int Add(const chunk_id, THashMap<T1, T2, T3, T4, T5>* pHash) { + template <class T1, class T2, class T3, class T4, class T5> + int Add(const chunk_id, THashMap<T1, T2, T3, T4, T5>* pHash) { DoAnyMap(*pHash); return 0; } - template <class T1, class T2, class T3, class T4, class T5> - int Add(const chunk_id, THashMultiMap<T1, T2, T3, T4, T5>* pHash) { + template <class T1, class T2, class T3, class T4, class T5> + int Add(const chunk_id, THashMultiMap<T1, T2, T3, T4, T5>* pHash) { DoAnyMultiMap(*pHash); return 0; } - template <class K, class L, class A> - int Add(const chunk_id, TSet<K, L, A>* pSet) { + template <class K, class L, class A> + int Add(const chunk_id, TSet<K, L, A>* pSet) { DoAnySet(*pSet); return 0; } - template <class T1, class T2, class T3, class T4> - int Add(const chunk_id, THashSet<T1, T2, T3, T4>* pHash) { + template <class T1, class T2, class T3, class T4> + int Add(const chunk_id, THashSet<T1, T2, T3, T4>* pHash) { DoAnySet(*pHash); return 0; } - - template <class T1> - int Add(const chunk_id, TArray2D<T1>* pArr) { + + template <class T1> + int Add(const chunk_id, TArray2D<T1>* pArr) { if (HasNonTrivialSerializer<T1>(0u)) Do2DArray(*pArr); else Do2DArrayData(*pArr); return 0; } - template <class T1> - int Add(const chunk_id, TList<T1>* pList) { - TList<T1>& data = *pList; - if (IsReading()) { + template <class T1> + int Add(const chunk_id, TList<T1>* pList) { + TList<T1>& data = *pList; + if (IsReading()) { int nSize; Add(2, &nSize); data.clear(); data.insert(data.begin(), nSize, T1()); - } else { + } else { int nSize = data.size(); Add(2, &nSize); } @@ -394,38 +394,38 @@ public: Add(i + 2, &(*k)); return 0; } - template <class T1, class T2> - int Add(const chunk_id, std::pair<T1, T2>* pData) { + template <class T1, class T2> + int Add(const chunk_id, std::pair<T1, T2>* pData) { Add(1, &(pData->first)); Add(2, &(pData->second)); return 0; } - template <class T1, size_t N> - int Add(const chunk_id, std::array<T1, N>* pData) { + template <class T1, size_t N> + int Add(const chunk_id, std::array<T1, N>* pData) { if (HasNonTrivialSerializer<T1>(0u)) { for (size_t i = 0; i < N; ++i) Add(1, &(*pData)[i]); } else { - DataChunk((void*)pData->data(), pData->size() * sizeof(T1)); + DataChunk((void*)pData->data(), pData->size() * sizeof(T1)); } return 0; } template <size_t N> - int Add(const chunk_id, std::bitset<N>* pData) { + int Add(const chunk_id, std::bitset<N>* pData) { if (IsReading()) { std::string s; Add(1, &s); *pData = std::bitset<N>(s); } else { - std::string s = pData->template to_string<char, std::char_traits<char>, std::allocator<char>>(); + std::string s = pData->template to_string<char, std::char_traits<char>, std::allocator<char>>(); Add(1, &s); } return 0; } - int Add(const chunk_id, TDynBitMap* pData) { + int Add(const chunk_id, TDynBitMap* pData) { if (IsReading()) { ui64 count = 0; Add(1, &count); @@ -491,17 +491,17 @@ public: } - void AddPolymorphicBase(chunk_id, IObjectBase* pObject) { + void AddPolymorphicBase(chunk_id, IObjectBase* pObject) { (*pObject) & (*this); } - template <class T1, class T2> - void DoPtr(TPtrBase<T1, T2>* pData) { + template <class T1, class T2> + void DoPtr(TPtrBase<T1, T2>* pData) { if (pData && pData->Get()) { } - if (IsReading()) + if (IsReading()) pData->Set(CastToUserObject(LoadObject(), (T1*)nullptr)); - else + else StoreObject(pData->GetBarePtr()); } template <class T, class TPolicy> @@ -525,52 +525,52 @@ public: } template <typename TOne> - void AddMulti(TOne& one) { + void AddMulti(TOne& one) { Add(0, &one); } - template <typename THead, typename... TTail> - void AddMulti(THead& head, TTail&... tail) { + template <typename THead, typename... TTail> + void AddMulti(THead& head, TTail&... tail) { Add(0, &head); - AddMulti(tail...); + AddMulti(tail...); } - template <class T, typename = decltype(std::declval<T&>() & std::declval<IBinSaver&>())> + template <class T, typename = decltype(std::declval<T&>() & std::declval<IBinSaver&>())> static bool HasNonTrivialSerializer(ui32) { return true; } - template <class T> + template <class T> static bool HasNonTrivialSerializer(...) { return sizeof(std::declval<IBinSaver*>()->Add(0, std::declval<T*>())) != 1; } - + public: - IBinSaver(IBinaryStream& stream, bool _bRead, bool stableOutput = false) - : bRead(_bRead) - , File(_bRead, stream) - , StableOutput(stableOutput) - { - } + IBinSaver(IBinaryStream& stream, bool _bRead, bool stableOutput = false) + : bRead(_bRead) + , File(_bRead, stream) + , StableOutput(stableOutput) + { + } virtual ~IBinSaver(); - bool IsValid() const { - return File.IsValid(); - } + bool IsValid() const { + return File.IsValid(); + } }; // realisation of forward declared serialisation operator -template <class TUserObj, class TRef> -int TPtrBase<TUserObj, TRef>::operator&(IBinSaver& f) { +template <class TUserObj, class TRef> +int TPtrBase<TUserObj, TRef>::operator&(IBinSaver& f) { f.DoPtr(this); return 0; } //////////////////////////////////////////////////////////////////////////////////////////////////// -extern TClassFactory<IObjectBase>* pSaverClasses; +extern TClassFactory<IObjectBase>* pSaverClasses; void StartRegisterSaveload(); -template <class TReg> +template <class TReg> struct TRegisterSaveLoadType { TRegisterSaveLoadType(int num) { StartRegisterSaveload(); @@ -578,52 +578,52 @@ struct TRegisterSaveLoadType { } }; -#define Y_BINSAVER_REGISTER(name) \ - BASIC_REGISTER_CLASS(name) \ +#define Y_BINSAVER_REGISTER(name) \ + BASIC_REGISTER_CLASS(name) \ static TRegisterSaveLoadType<name> init##name(MurmurHash<int>(#name, sizeof(#name))); -#define REGISTER_SAVELOAD_CLASS(N, name) \ - BASIC_REGISTER_CLASS(name) \ +#define REGISTER_SAVELOAD_CLASS(N, name) \ + BASIC_REGISTER_CLASS(name) \ static TRegisterSaveLoadType<name> init##name##N(N); // using TObj/TRef on forward declared templ class will not work // but multiple registration with same id is allowed -#define REGISTER_SAVELOAD_TEMPL1_CLASS(N, className, T) \ - static TRegisterSaveLoadType<className<T>> init##className##T##N(N); +#define REGISTER_SAVELOAD_TEMPL1_CLASS(N, className, T) \ + static TRegisterSaveLoadType<className<T>> init##className##T##N(N); -#define REGISTER_SAVELOAD_TEMPL2_CLASS(N, className, T1, T2) \ - typedef className<T1, T2> temp##className##T1##_##T2##temp; \ - static TRegisterSaveLoadType<className<T1, T2>> init##className##T1##_##T2##N(N); +#define REGISTER_SAVELOAD_TEMPL2_CLASS(N, className, T1, T2) \ + typedef className<T1, T2> temp##className##T1##_##T2##temp; \ + static TRegisterSaveLoadType<className<T1, T2>> init##className##T1##_##T2##N(N); -#define REGISTER_SAVELOAD_TEMPL3_CLASS(N, className, T1, T2, T3) \ - typedef className<T1, T2, T3> temp##className##T1##_##T2##_##T3##temp; \ - static TRegisterSaveLoadType<className<T1, T2, T3>> init##className##T1##_##T2##_##T3##N(N); +#define REGISTER_SAVELOAD_TEMPL3_CLASS(N, className, T1, T2, T3) \ + typedef className<T1, T2, T3> temp##className##T1##_##T2##_##T3##temp; \ + static TRegisterSaveLoadType<className<T1, T2, T3>> init##className##T1##_##T2##_##T3##N(N); -#define REGISTER_SAVELOAD_NM_CLASS(N, nmspace, className) \ - BASIC_REGISTER_CLASS(nmspace::className) \ +#define REGISTER_SAVELOAD_NM_CLASS(N, nmspace, className) \ + BASIC_REGISTER_CLASS(nmspace::className) \ static TRegisterSaveLoadType<nmspace::className> init_##nmspace##_##name##N(N); #define REGISTER_SAVELOAD_NM2_CLASS(N, nmspace1, nmspace2, className) \ BASIC_REGISTER_CLASS(nmspace1::nmspace2::className) \ static TRegisterSaveLoadType<nmspace1::nmspace2::className> init_##nmspace1##_##nmspace2##_##name##N(N); -#define REGISTER_SAVELOAD_TEMPL1_NM_CLASS(N, nmspace, className, T) \ - typedef nmspace::className<T> temp_init##nmspace##className##T##temp; \ - BASIC_REGISTER_CLASS(nmspace::className<T>) \ - static TRegisterSaveLoadType<nmspace::className<T>> temp_init##nmspace##_##name##T##N(N); +#define REGISTER_SAVELOAD_TEMPL1_NM_CLASS(N, nmspace, className, T) \ + typedef nmspace::className<T> temp_init##nmspace##className##T##temp; \ + BASIC_REGISTER_CLASS(nmspace::className<T>) \ + static TRegisterSaveLoadType<nmspace::className<T>> temp_init##nmspace##_##name##T##N(N); #define REGISTER_SAVELOAD_CLASS_NAME(N, cls, name) \ - BASIC_REGISTER_CLASS(cls) \ - static TRegisterSaveLoadType<cls> init##name##N(N); + BASIC_REGISTER_CLASS(cls) \ + static TRegisterSaveLoadType<cls> init##name##N(N); #define REGISTER_SAVELOAD_CLASS_NS_PREF(N, cls, ns, pref) \ - REGISTER_SAVELOAD_CLASS_NAME(N, ns ::cls, _##pref##_##cls) + REGISTER_SAVELOAD_CLASS_NAME(N, ns ::cls, _##pref##_##cls) -#define SAVELOAD(...) \ - int operator&(IBinSaver& f) { \ - f.AddMulti(__VA_ARGS__); \ - return 0; \ - } +#define SAVELOAD(...) \ + int operator&(IBinSaver& f) { \ + f.AddMulti(__VA_ARGS__); \ + return 0; \ + } #define SAVELOAD_OVERRIDE_WITHOUT_BASE(...) \ int operator&(IBinSaver& f) override { \ @@ -638,9 +638,9 @@ struct TRegisterSaveLoadType { return 0; \ } -#define SAVELOAD_BASE(...) \ - int operator&(IBinSaver& f) { \ - TBase::operator&(f); \ - f.AddMulti(__VA_ARGS__); \ - return 0; \ +#define SAVELOAD_BASE(...) \ + int operator&(IBinSaver& f) { \ + TBase::operator&(f); \ + f.AddMulti(__VA_ARGS__); \ + return 0; \ } diff --git a/library/cpp/binsaver/blob_io.h b/library/cpp/binsaver/blob_io.h index abe518ef30..efde51fe88 100644 --- a/library/cpp/binsaver/blob_io.h +++ b/library/cpp/binsaver/blob_io.h @@ -5,15 +5,15 @@ #include <util/memory/blob.h> -class TYaBlobStream: public IBinaryStream { +class TYaBlobStream: public IBinaryStream { TBlob Blob; i64 Pos; - int WriteImpl(const void*, int) override { + int WriteImpl(const void*, int) override { Y_ASSERT(0); return 0; } - int ReadImpl(void* userBuffer, int size) override { + int ReadImpl(void* userBuffer, int size) override { if (size == 0) return 0; i64 res = Min<i64>(Blob.Length() - Pos, size); @@ -22,23 +22,23 @@ class TYaBlobStream: public IBinaryStream { Pos += res; return res; } - bool IsValid() const override { - return true; - } - bool IsFailed() const override { - return false; - } - + bool IsValid() const override { + return true; + } + bool IsFailed() const override { + return false; + } + public: - TYaBlobStream(const TBlob& blob) - : Blob(blob) - , Pos(0) - { - } + TYaBlobStream(const TBlob& blob) + : Blob(blob) + , Pos(0) + { + } }; -template <class T> -inline void SerializeBlob(const TBlob& data, T& c) { +template <class T> +inline void SerializeBlob(const TBlob& data, T& c) { TYaBlobStream f(data); { IBinSaver bs(f, true); diff --git a/library/cpp/binsaver/buffered_io.cpp b/library/cpp/binsaver/buffered_io.cpp index dd88b04bc5..81ca656bf3 100644 --- a/library/cpp/binsaver/buffered_io.cpp +++ b/library/cpp/binsaver/buffered_io.cpp @@ -1,6 +1,6 @@ #include "buffered_io.h" -i64 IBinaryStream::LongWrite(const void* userBuffer, i64 size) { +i64 IBinaryStream::LongWrite(const void* userBuffer, i64 size) { Y_VERIFY(size >= 0, "IBinaryStream::Write() called with a negative buffer size."); i64 leftToWrite = size; @@ -18,7 +18,7 @@ i64 IBinaryStream::LongWrite(const void* userBuffer, i64 size) { return size - leftToWrite; } -i64 IBinaryStream::LongRead(void* userBuffer, i64 size) { +i64 IBinaryStream::LongRead(void* userBuffer, i64 size) { Y_VERIFY(size >= 0, "IBinaryStream::Read() called with a negative buffer size."); i64 leftToRead = size; diff --git a/library/cpp/binsaver/buffered_io.h b/library/cpp/binsaver/buffered_io.h index 75465c9c5c..893f93cddd 100644 --- a/library/cpp/binsaver/buffered_io.h +++ b/library/cpp/binsaver/buffered_io.h @@ -1,15 +1,15 @@ #pragma once - + #include <util/system/yassert.h> #include <util/generic/utility.h> #include <util/generic/ylimits.h> #include <string.h> -struct IBinaryStream { - virtual ~IBinaryStream() = default; - ; +struct IBinaryStream { + virtual ~IBinaryStream() = default; + ; - inline i64 Write(const void* userBuffer, i64 size) { + inline i64 Write(const void* userBuffer, i64 size) { if (size <= Max<int>()) { return WriteImpl(userBuffer, static_cast<int>(size)); } else { @@ -17,7 +17,7 @@ struct IBinaryStream { } } - inline i64 Read(void* userBuffer, i64 size) { + inline i64 Read(void* userBuffer, i64 size) { if (size <= Max<int>()) { return ReadImpl(userBuffer, static_cast<int>(size)); } else { @@ -29,26 +29,26 @@ struct IBinaryStream { virtual bool IsFailed() const = 0; private: - virtual int WriteImpl(const void* userBuffer, int size) = 0; - virtual int ReadImpl(void* userBuffer, int size) = 0; + virtual int WriteImpl(const void* userBuffer, int size) = 0; + virtual int ReadImpl(void* userBuffer, int size) = 0; - i64 LongRead(void* userBuffer, i64 size); - i64 LongWrite(const void* userBuffer, i64 size); + i64 LongRead(void* userBuffer, i64 size); + i64 LongWrite(const void* userBuffer, i64 size); }; template <int N_SIZE = 16384> -class TBufferedStream { +class TBufferedStream { char Buf[N_SIZE]; i64 Pos, BufSize; - IBinaryStream& Stream; + IBinaryStream& Stream; bool bIsReading, bIsEof, bFailed; - void ReadComplex(void* userBuffer, i64 size) { + void ReadComplex(void* userBuffer, i64 size) { if (bIsEof) { memset(userBuffer, 0, size); return; } - char* dst = (char*)userBuffer; + char* dst = (char*)userBuffer; i64 leftBytes = BufSize - Pos; memcpy(dst, Buf + Pos, leftBytes); dst += leftBytes; @@ -70,28 +70,28 @@ class TBufferedStream { } } - void WriteComplex(const void* userBuffer, i64 size) { + void WriteComplex(const void* userBuffer, i64 size) { Flush(); if (size >= N_SIZE) { Stream.Write(userBuffer, size); bFailed = Stream.IsFailed(); - } else + } else Write(userBuffer, size); } - void operator=(const TBufferedStream&) { - } - + void operator=(const TBufferedStream&) { + } + public: - TBufferedStream(bool bRead, IBinaryStream& stream) - : Pos(0) - , BufSize(0) - , Stream(stream) - , bIsReading(bRead) - , bIsEof(false) - , bFailed(false) - { - } + TBufferedStream(bool bRead, IBinaryStream& stream) + : Pos(0) + , BufSize(0) + , Stream(stream) + , bIsReading(bRead) + , bIsEof(false) + , bFailed(false) + { + } ~TBufferedStream() { if (!bIsReading) Flush(); @@ -104,10 +104,10 @@ public: bFailed = Stream.IsFailed(); Pos = 0; } - bool IsEof() const { - return bIsEof; - } - inline void Read(void* userBuffer, i64 size) { + bool IsEof() const { + return bIsEof; + } + inline void Read(void* userBuffer, i64 size) { Y_ASSERT(bIsReading); if (!bIsEof && size + Pos <= BufSize) { memcpy(userBuffer, Buf + Pos, size); @@ -116,7 +116,7 @@ public: } ReadComplex(userBuffer, size); } - inline void Write(const void* userBuffer, i64 size) { + inline void Write(const void* userBuffer, i64 size) { Y_ASSERT(!bIsReading); if (Pos + size < N_SIZE) { memcpy(Buf + Pos, userBuffer, size); @@ -125,10 +125,10 @@ public: } WriteComplex(userBuffer, size); } - bool IsValid() const { - return Stream.IsValid(); - } - bool IsFailed() const { - return bFailed; - } + bool IsValid() const { + return Stream.IsValid(); + } + bool IsFailed() const { + return bFailed; + } }; diff --git a/library/cpp/binsaver/class_factory.h b/library/cpp/binsaver/class_factory.h index e83512331b..f01f2c33a3 100644 --- a/library/cpp/binsaver/class_factory.h +++ b/library/cpp/binsaver/class_factory.h @@ -1,5 +1,5 @@ #pragma once - + #include <typeinfo> #include <util/generic/hash.h> #include <util/generic/vector.h> @@ -10,96 +10,96 @@ // factory is using RTTI // objects should inherit T and T must have at least 1 virtual function template <class T> -class TClassFactory { +class TClassFactory { public: - typedef const std::type_info* VFT; - + typedef const std::type_info* VFT; + private: typedef T* (*newFunc)(); - typedef THashMap<int, newFunc> CTypeNewHash; // typeID->newFunc() + typedef THashMap<int, newFunc> CTypeNewHash; // typeID->newFunc() typedef THashMap<VFT, int> CTypeIndexHash; // vftable->typeID CTypeIndexHash typeIndex; CTypeNewHash typeInfo; - void RegisterTypeBase(int nTypeID, newFunc func, VFT vft); - static VFT GetObjectType(T* pObject) { - return &typeid(*pObject); - } - int VFT2TypeID(VFT t) { - CTypeIndexHash::iterator i = typeIndex.find(t); - if (i != typeIndex.end()) + void RegisterTypeBase(int nTypeID, newFunc func, VFT vft); + static VFT GetObjectType(T* pObject) { + return &typeid(*pObject); + } + int VFT2TypeID(VFT t) { + CTypeIndexHash::iterator i = typeIndex.find(t); + if (i != typeIndex.end()) return i->second; - for (i = typeIndex.begin(); i != typeIndex.end(); ++i) { - if (*i->first == *t) { + for (i = typeIndex.begin(); i != typeIndex.end(); ++i) { + if (*i->first == *t) { typeIndex[t] = i->second; return i->second; } } return -1; } - + public: - template <class TT> - void RegisterType(int nTypeID, newFunc func, TT*) { - RegisterTypeBase(nTypeID, func, &typeid(TT)); - } - void RegisterTypeSafe(int nTypeID, newFunc func) { + template <class TT> + void RegisterType(int nTypeID, newFunc func, TT*) { + RegisterTypeBase(nTypeID, func, &typeid(TT)); + } + void RegisterTypeSafe(int nTypeID, newFunc func) { TPtr<T> pObj = func(); - VFT vft = GetObjectType(pObj); - RegisterTypeBase(nTypeID, func, vft); - } - T* CreateObject(int nTypeID) { - newFunc f = typeInfo[nTypeID]; - if (f) - return f(); - return nullptr; - } - int GetObjectTypeID(T* pObject) { - return VFT2TypeID(GetObjectType(pObject)); - } - template <class TT> - int GetTypeID(TT* p = 0) { - (void)p; - return VFT2TypeID(&typeid(TT)); + VFT vft = GetObjectType(pObj); + RegisterTypeBase(nTypeID, func, vft); } + T* CreateObject(int nTypeID) { + newFunc f = typeInfo[nTypeID]; + if (f) + return f(); + return nullptr; + } + int GetObjectTypeID(T* pObject) { + return VFT2TypeID(GetObjectType(pObject)); + } + template <class TT> + int GetTypeID(TT* p = 0) { + (void)p; + return VFT2TypeID(&typeid(TT)); + } - void GetAllTypeIDs(TVector<int>& typeIds) const { + void GetAllTypeIDs(TVector<int>& typeIds) const { typeIds.clear(); - for (typename CTypeNewHash::const_iterator iter = typeInfo.begin(); - iter != typeInfo.end(); - ++iter) { - typeIds.push_back(iter->first); + for (typename CTypeNewHash::const_iterator iter = typeInfo.begin(); + iter != typeInfo.end(); + ++iter) { + typeIds.push_back(iter->first); } } }; //////////////////////////////////////////////////////////////////////////////////////////////////// template <class T> -void TClassFactory<T>::RegisterTypeBase(int nTypeID, newFunc func, VFT vft) { - if (typeInfo.find(nTypeID) != typeInfo.end()) { +void TClassFactory<T>::RegisterTypeBase(int nTypeID, newFunc func, VFT vft) { + if (typeInfo.find(nTypeID) != typeInfo.end()) { TObj<IObjectBase> o1 = typeInfo[nTypeID](); TObj<IObjectBase> o2 = func(); - - // stupid clang warning - auto& o1v = *o1; - auto& o2v = *o2; - - if (typeid(o1v) != typeid(o2v)) { - fprintf(stderr, "IBinSaver: Type ID 0x%08X has been already used\n", nTypeID); + + // stupid clang warning + auto& o1v = *o1; + auto& o2v = *o2; + + if (typeid(o1v) != typeid(o2v)) { + fprintf(stderr, "IBinSaver: Type ID 0x%08X has been already used\n", nTypeID); abort(); } } - CTypeIndexHash::iterator typeIndexIt = typeIndex.find(vft); - if (typeIndexIt != typeIndex.end() && nTypeID != typeIndexIt->second) { - fprintf(stderr, "IBinSaver: class (Type ID 0x%08X) has been already registered (Type ID 0x%08X)\n", nTypeID, typeIndexIt->second); + CTypeIndexHash::iterator typeIndexIt = typeIndex.find(vft); + if (typeIndexIt != typeIndex.end() && nTypeID != typeIndexIt->second) { + fprintf(stderr, "IBinSaver: class (Type ID 0x%08X) has been already registered (Type ID 0x%08X)\n", nTypeID, typeIndexIt->second); abort(); } - typeIndex[vft] = nTypeID; - typeInfo[nTypeID] = func; + typeIndex[vft] = nTypeID; + typeInfo[nTypeID] = func; } //////////////////////////////////////////////////////////////////////////////////////////////////// // macro for registering CFundament derivatives -#define REGISTER_CLASS(factory, N, name) factory.RegisterType(N, name::New##name, (name*)0); -#define REGISTER_TEMPL_CLASS(factory, N, name, className) factory.RegisterType(N, name::New##className, (name*)0); -#define REGISTER_CLASS_NM(factory, N, name, nmspace) factory.RegisterType(N, nmspace::name::New##name, (nmspace::name*)0); +#define REGISTER_CLASS(factory, N, name) factory.RegisterType(N, name::New##name, (name*)0); +#define REGISTER_TEMPL_CLASS(factory, N, name, className) factory.RegisterType(N, name::New##className, (name*)0); +#define REGISTER_CLASS_NM(factory, N, name, nmspace) factory.RegisterType(N, nmspace::name::New##name, (nmspace::name*)0); diff --git a/library/cpp/binsaver/ut/binsaver_ut.cpp b/library/cpp/binsaver/ut/binsaver_ut.cpp index 37eba5406f..e4237e0875 100644 --- a/library/cpp/binsaver/ut/binsaver_ut.cpp +++ b/library/cpp/binsaver/ut/binsaver_ut.cpp @@ -27,14 +27,14 @@ struct TCustomOuterSerializer { TString StrData; }; -void operator&(TCustomOuterSerializer& s, IBinSaver& f); +void operator&(TCustomOuterSerializer& s, IBinSaver& f); struct TCustomOuterSerializerTmpl { ui32 Data = 0; TString StrData; }; -struct TCustomOuterSerializerTmplDerived: public TCustomOuterSerializerTmpl { +struct TCustomOuterSerializerTmplDerived: public TCustomOuterSerializerTmpl { ui32 Data = 0; TString StrData; }; @@ -45,7 +45,7 @@ struct TMoveOnlyType { TMoveOnlyType() = default; TMoveOnlyType(TMoveOnlyType&&) = default; - bool operator==(const TMoveOnlyType& obj) const { + bool operator==(const TMoveOnlyType& obj) const { return Data == obj.Data; } }; @@ -55,13 +55,13 @@ struct TTypeWithArray { TString Array[2][2]{{"test", "data"}, {"and", "more"}}; SAVELOAD(Data, Array); - bool operator==(const TTypeWithArray& obj) const { - return Data == obj.Data && std::equal(std::begin(Array[0]), std::end(Array[0]), obj.Array[0]) && std::equal(std::begin(Array[1]), std::end(Array[1]), obj.Array[1]); + bool operator==(const TTypeWithArray& obj) const { + return Data == obj.Data && std::equal(std::begin(Array[0]), std::end(Array[0]), obj.Array[0]) && std::equal(std::begin(Array[1]), std::end(Array[1]), obj.Array[1]); } }; -template <typename T, typename = std::enable_if_t<std::is_base_of<TCustomOuterSerializerTmpl, T>::value>> -int operator&(T& s, IBinSaver& f); +template <typename T, typename = std::enable_if_t<std::is_base_of<TCustomOuterSerializerTmpl, T>::value>> +int operator&(T& s, IBinSaver& f); static bool operator==(const TBlob& l, const TBlob& r) { return TStringBuf(l.AsCharPtr(), l.Size()) == TStringBuf(r.AsCharPtr(), r.Size()); @@ -69,45 +69,45 @@ static bool operator==(const TBlob& l, const TBlob& r) { Y_UNIT_TEST_SUITE(BinSaver){ Y_UNIT_TEST(HasTrivialSerializer){ - UNIT_ASSERT(!IBinSaver::HasNonTrivialSerializer<TBinarySerializable>(0u)); -UNIT_ASSERT(!IBinSaver::HasNonTrivialSerializer<TNonBinarySerializable>(0u)); -UNIT_ASSERT(IBinSaver::HasNonTrivialSerializer<TCustomSerializer>(0u)); -UNIT_ASSERT(IBinSaver::HasNonTrivialSerializer<TCustomOuterSerializer>(0u)); -UNIT_ASSERT(IBinSaver::HasNonTrivialSerializer<TCustomOuterSerializerTmpl>(0u)); -UNIT_ASSERT(IBinSaver::HasNonTrivialSerializer<TCustomOuterSerializerTmplDerived>(0u)); -UNIT_ASSERT(IBinSaver::HasNonTrivialSerializer<TVector<TCustomSerializer>>(0u)); -} + UNIT_ASSERT(!IBinSaver::HasNonTrivialSerializer<TBinarySerializable>(0u)); +UNIT_ASSERT(!IBinSaver::HasNonTrivialSerializer<TNonBinarySerializable>(0u)); +UNIT_ASSERT(IBinSaver::HasNonTrivialSerializer<TCustomSerializer>(0u)); +UNIT_ASSERT(IBinSaver::HasNonTrivialSerializer<TCustomOuterSerializer>(0u)); +UNIT_ASSERT(IBinSaver::HasNonTrivialSerializer<TCustomOuterSerializerTmpl>(0u)); +UNIT_ASSERT(IBinSaver::HasNonTrivialSerializer<TCustomOuterSerializerTmplDerived>(0u)); +UNIT_ASSERT(IBinSaver::HasNonTrivialSerializer<TVector<TCustomSerializer>>(0u)); +} Y_UNIT_TEST(TestStroka) { TestBinSaverSerialization(TString("QWERTY")); -} +} Y_UNIT_TEST(TestMoveOnlyType) { TestBinSaverSerializationToBuffer(TMoveOnlyType()); -} +} Y_UNIT_TEST(TestVectorStrok) { TestBinSaverSerialization(TVector<TString>{"A", "B", "C"}); -} +} Y_UNIT_TEST(TestCArray) { TestBinSaverSerialization(TTypeWithArray()); -} +} Y_UNIT_TEST(TestSets) { TestBinSaverSerialization(THashSet<TString>{"A", "B", "C"}); TestBinSaverSerialization(TSet<TString>{"A", "B", "C"}); -} +} Y_UNIT_TEST(TestMaps) { TestBinSaverSerialization(THashMap<TString, ui32>{{"A", 1}, {"B", 2}, {"C", 3}}); TestBinSaverSerialization(TMap<TString, ui32>{{"A", 1}, {"B", 2}, {"C", 3}}); -} +} Y_UNIT_TEST(TestBlob) { TestBinSaverSerialization(TBlob::FromStringSingleThreaded("qwerty")); -} +} Y_UNIT_TEST(TestVariant) { { @@ -126,73 +126,73 @@ Y_UNIT_TEST(TestVariant) { } Y_UNIT_TEST(TestPod) { - struct TPod { - ui32 A = 5; - ui64 B = 7; - bool operator==(const TPod& other) const { - return A == other.A && B == other.B; - } - }; + struct TPod { + ui32 A = 5; + ui64 B = 7; + bool operator==(const TPod& other) const { + return A == other.A && B == other.B; + } + }; TestBinSaverSerialization(TPod()); - TPod custom; - custom.A = 25; - custom.B = 37; + TPod custom; + custom.A = 25; + custom.B = 37; TestBinSaverSerialization(custom); TestBinSaverSerialization(TVector<TPod>{custom}); -} +} Y_UNIT_TEST(TestSubPod) { - struct TPod { - struct TSub { - ui32 X = 10; - bool operator==(const TSub& other) const { - return X == other.X; + struct TPod { + struct TSub { + ui32 X = 10; + bool operator==(const TSub& other) const { + return X == other.X; } }; - TVector<TSub> B; - int operator&(IBinSaver& f) { - f.Add(0, &B); - return 0; - } - bool operator==(const TPod& other) const { - return B == other.B; - } - }; + TVector<TSub> B; + int operator&(IBinSaver& f) { + f.Add(0, &B); + return 0; + } + bool operator==(const TPod& other) const { + return B == other.B; + } + }; TestBinSaverSerialization(TPod()); - TPod::TSub sub; - sub.X = 1; - TPod custom; - custom.B = {sub}; + TPod::TSub sub; + sub.X = 1; + TPod custom; + custom.B = {sub}; TestBinSaverSerialization(TVector<TPod>{custom}); -} +} Y_UNIT_TEST(TestMemberAndOpIsMain) { - struct TBase { - TString S; - virtual int operator&(IBinSaver& f) { - f.Add(0, &S); - return 0; - } - virtual ~TBase() = default; - }; - - struct TDerived: public TBase { - int A = 0; - int operator&(IBinSaver& f)override { - f.Add(0, static_cast<TBase*>(this)); - f.Add(0, &A); - return 0; - } - bool operator==(const TDerived& other) const { - return A == other.A && S == other.S; - } - }; - - TDerived obj; - obj.S = "TString"; - obj.A = 42; + struct TBase { + TString S; + virtual int operator&(IBinSaver& f) { + f.Add(0, &S); + return 0; + } + virtual ~TBase() = default; + }; + + struct TDerived: public TBase { + int A = 0; + int operator&(IBinSaver& f)override { + f.Add(0, static_cast<TBase*>(this)); + f.Add(0, &A); + return 0; + } + bool operator==(const TDerived& other) const { + return A == other.A && S == other.S; + } + }; + + TDerived obj; + obj.S = "TString"; + obj.A = 42; TestBinSaverSerialization(obj); -} -} -; +} +} +; diff --git a/library/cpp/binsaver/util_stream_io.h b/library/cpp/binsaver/util_stream_io.h index d65d630b93..bc560cd5c3 100644 --- a/library/cpp/binsaver/util_stream_io.h +++ b/library/cpp/binsaver/util_stream_io.h @@ -1,37 +1,37 @@ #pragma once - + #include "bin_saver.h" #include <util/stream/input.h> #include <util/stream/output.h> #include <util/stream/file.h> -class TYaStreamInput: public IBinaryStream { - IInputStream& Stream; +class TYaStreamInput: public IBinaryStream { + IInputStream& Stream; - int WriteImpl(const void*, int) override { + int WriteImpl(const void*, int) override { Y_ASSERT(0); return 0; } - int ReadImpl(void* userBuffer, int size) override { - return (int)Stream.Read(userBuffer, (size_t)size); - } - bool IsValid() const override { - return true; - } - bool IsFailed() const override { - return false; + int ReadImpl(void* userBuffer, int size) override { + return (int)Stream.Read(userBuffer, (size_t)size); } - + bool IsValid() const override { + return true; + } + bool IsFailed() const override { + return false; + } + public: - TYaStreamInput(IInputStream& stream) - : Stream(stream) - { + TYaStreamInput(IInputStream& stream) + : Stream(stream) + { } }; -template <class T> -inline void SerializeFromStream(IInputStream& stream, T& c) { +template <class T> +inline void SerializeFromStream(IInputStream& stream, T& c) { TYaStreamInput f(stream); { IBinSaver bs(f, true); @@ -39,38 +39,38 @@ inline void SerializeFromStream(IInputStream& stream, T& c) { } } -template <class T> -inline void SerializeFromFile(const TString& fileName, T& c) { +template <class T> +inline void SerializeFromFile(const TString& fileName, T& c) { TIFStream in(fileName); SerializeFromStream(in, c); } -class TYaStreamOutput: public IBinaryStream { - IOutputStream& Stream; +class TYaStreamOutput: public IBinaryStream { + IOutputStream& Stream; - int WriteImpl(const void* what, int size) override { - Stream.Write(what, (size_t)size); + int WriteImpl(const void* what, int size) override { + Stream.Write(what, (size_t)size); return size; } - int ReadImpl(void*, int) override { + int ReadImpl(void*, int) override { Y_ASSERT(0); return 0; } - bool IsValid() const override { - return true; - } - bool IsFailed() const override { - return false; - } - + bool IsValid() const override { + return true; + } + bool IsFailed() const override { + return false; + } + public: - TYaStreamOutput(IOutputStream& stream) - : Stream(stream) - { + TYaStreamOutput(IOutputStream& stream) + : Stream(stream) + { } }; -template <class T> +template <class T> inline void SerializeToArcadiaStream(IOutputStream& stream, T& c) { TYaStreamOutput f(stream); { @@ -79,8 +79,8 @@ inline void SerializeToArcadiaStream(IOutputStream& stream, T& c) { } } -template <class T> -inline void SerializeToFile(const TString& fileName, T& c) { +template <class T> +inline void SerializeToFile(const TString& fileName, T& c) { TOFStream out(fileName); SerializeToArcadiaStream(out, c); } diff --git a/library/cpp/binsaver/ya.make b/library/cpp/binsaver/ya.make index 9693c54639..417055decf 100644 --- a/library/cpp/binsaver/ya.make +++ b/library/cpp/binsaver/ya.make @@ -1,7 +1,7 @@ LIBRARY() -OWNER(gulin) - +OWNER(gulin) + SRCS( class_factory.h bin_saver.cpp diff --git a/library/cpp/bit_io/bitinout_ut.cpp b/library/cpp/bit_io/bitinout_ut.cpp index 23a1ddf344..da71d68070 100644 --- a/library/cpp/bit_io/bitinout_ut.cpp +++ b/library/cpp/bit_io/bitinout_ut.cpp @@ -3,16 +3,16 @@ #include <library/cpp/testing/unittest/registar.h> -#include <util/stream/buffer.h> +#include <util/stream/buffer.h> #include <util/generic/buffer.h> namespace NBitIO { static const char BITS_REF[] = - "00100010 01000000 00000000 00100111 11011111 01100111 11010101 00010100 " - "00100010 01100011 11100011 00110000 11011011 11011111 01001100 00110101 " - "10011110 01011111 01010000 00000110 00011011 00100110 00010100 01110011 " - "00001010 10101010 10101010 10101010 10101010 10101010 10101010 10101010 " - "10110101 01010101 01010101 01010101 01010101 01010101 01010101 01010101 " + "00100010 01000000 00000000 00100111 11011111 01100111 11010101 00010100 " + "00100010 01100011 11100011 00110000 11011011 11011111 01001100 00110101 " + "10011110 01011111 01010000 00000110 00011011 00100110 00010100 01110011 " + "00001010 10101010 10101010 10101010 10101010 10101010 10101010 10101010 " + "10110101 01010101 01010101 01010101 01010101 01010101 01010101 01010101 " "01000000"; inline ui64 Bits(ui64 bytes) { @@ -47,7 +47,7 @@ namespace NBitIO { class TBitIOTest: public TTestBase { UNIT_TEST_SUITE(TBitIOTest); - UNIT_TEST(TestBitIO) + UNIT_TEST(TestBitIO) UNIT_TEST_SUITE_END(); private: @@ -59,13 +59,13 @@ private: } void DoRead(TBi& b, ui32& t) { - b.Read(t, 1, 0); // 1 + b.Read(t, 1, 0); // 1 b.ReadK<3>(t, 1); // 4 - b.Read(t, 5, 4); // 9 + b.Read(t, 5, 4); // 9 b.ReadK<14>(t, 9); // 23 - b.Read(t, 1, 23); // 24 + b.Read(t, 1, 23); // 24 b.ReadK<5>(t, 24); // 29 - b.Read(t, 3, 29); // 32 + b.Read(t, 3, 29); // 32 } template <typename TBo> @@ -76,7 +76,7 @@ private: b.Write(t, 14, 9); //23 b.Write(t, 1, 23); //24 b.Write(t, 5, 24); //29 - b.Write(t, 3, 29); //32 + b.Write(t, 3, 29); //32 } template <typename TBo> @@ -213,7 +213,7 @@ private: DoRead(in, v); UNIT_ASSERT_VALUES_EQUAL_C(v, 1637415112ul, (rem + ": " + NBitIO::PrintBits(v))); DoRead(in, v); - UNIT_ASSERT_VALUES_EQUAL_C(v, 897998715u, (rem + ": " + NBitIO::PrintBits(v))); + UNIT_ASSERT_VALUES_EQUAL_C(v, 897998715u, (rem + ": " + NBitIO::PrintBits(v))); DoRead(in, v); UNIT_ASSERT_VALUES_EQUAL_C(v, 201416527u, (rem + ": " + NBitIO::PrintBits(v))); DoRead(in, v); diff --git a/library/cpp/bit_io/bitinput.cpp b/library/cpp/bit_io/bitinput.cpp index d7c37f06fc..a453450fff 100644 --- a/library/cpp/bit_io/bitinput.cpp +++ b/library/cpp/bit_io/bitinput.cpp @@ -1 +1 @@ -#include "bitinput.h" +#include "bitinput.h" diff --git a/library/cpp/bit_io/bitinput.h b/library/cpp/bit_io/bitinput.h index 85711eb7f9..ffd9e48f7b 100644 --- a/library/cpp/bit_io/bitinput.h +++ b/library/cpp/bit_io/bitinput.h @@ -2,170 +2,170 @@ #include "bitinput_impl.h" -#include <util/system/yassert.h> +#include <util/system/yassert.h> #include <util/generic/vector.h> #include <util/generic/yexception.h> #include <iterator> namespace NBitIO { - // Based on junk/solar/codecs/bitstream.h + // Based on junk/solar/codecs/bitstream.h - class TBitInput: protected TBitInputImpl { - public: - template <typename TVec> - explicit TBitInput(const TVec& vec) + class TBitInput: protected TBitInputImpl { + public: + template <typename TVec> + explicit TBitInput(const TVec& vec) : TBitInputImpl(std::begin(vec), std::end(vec)) - { - } + { + } - TBitInput(const char* start, const char* end) - : TBitInputImpl(start, end) - { - } + TBitInput(const char* start, const char* end) + : TBitInputImpl(start, end) + { + } - bool Eof() const { - return EofImpl(); - } + bool Eof() const { + return EofImpl(); + } - ui64 GetOffset() const { - ui64 bo = BitOffset(); - return bo / 8 + !!(bo % 8); - } + ui64 GetOffset() const { + ui64 bo = BitOffset(); + return bo / 8 + !!(bo % 8); + } using TBitInputImpl::GetBitLength; - ui64 GetBitOffset() const { - return BitOffset() % 8; - } + ui64 GetBitOffset() const { + return BitOffset() % 8; + } - public: - // Read with static number of bits. - // Preserves what's in result. - template <ui64 bits, typename T> + public: + // Read with static number of bits. + // Preserves what's in result. + template <ui64 bits, typename T> Y_FORCE_INLINE bool ReadK(T& result, ui64 skipbits) { - ui64 r64 = 0; - bool ret = bits <= 56 ? ReadKImpl<bits>(r64) : ReadSafe(r64, bits); - CopyToResultK<bits>(result, r64, skipbits); - return ret; - } - - // Read with static number of bits. - // Zeroes other bits in result. - template <ui64 bits, typename T> + ui64 r64 = 0; + bool ret = bits <= 56 ? ReadKImpl<bits>(r64) : ReadSafe(r64, bits); + CopyToResultK<bits>(result, r64, skipbits); + return ret; + } + + // Read with static number of bits. + // Zeroes other bits in result. + template <ui64 bits, typename T> Y_FORCE_INLINE bool ReadK(T& result) { - ui64 r = 0; - bool res = ReadK<bits>(r); - result = r; - return res; - } - - // Shortcut to impl. - template <ui64 bits> + ui64 r = 0; + bool res = ReadK<bits>(r); + result = r; + return res; + } + + // Shortcut to impl. + template <ui64 bits> Y_FORCE_INLINE bool ReadK(ui64& result) { - if (bits <= 56) - return ReadKImpl<bits>(result); + if (bits <= 56) + return ReadKImpl<bits>(result); - ui64 r1 = 0ULL; - ui64 r2 = 0ULL; + ui64 r1 = 0ULL; + ui64 r2 = 0ULL; - bool ret1 = ReadKImpl<56ULL>(r1); - bool ret2 = ReadKImpl<(bits > 56ULL ? bits - 56ULL : 0) /*or else we get negative param in template*/>(r2); + bool ret1 = ReadKImpl<56ULL>(r1); + bool ret2 = ReadKImpl<(bits > 56ULL ? bits - 56ULL : 0) /*or else we get negative param in template*/>(r2); - result = (r2 << 56ULL) | r1; + result = (r2 << 56ULL) | r1; - return ret1 & ret2; - } + return ret1 & ret2; + } - // It's safe to read up to 64 bits. - // Zeroes other bits in result. - template <typename T> + // It's safe to read up to 64 bits. + // Zeroes other bits in result. + template <typename T> Y_FORCE_INLINE bool ReadSafe(T& result, ui64 bits) { - if (bits <= 56ULL) - return Read(result, bits); + if (bits <= 56ULL) + return Read(result, bits); - ui64 r1 = 0ULL; - ui64 r2 = 0ULL; + ui64 r1 = 0ULL; + ui64 r2 = 0ULL; - bool ret1 = ReadKImpl<56ULL>(r1); - bool ret2 = ReadImpl(r2, bits - 56ULL); + bool ret1 = ReadKImpl<56ULL>(r1); + bool ret2 = ReadImpl(r2, bits - 56ULL); - result = (r2 << 56ULL) | r1; + result = (r2 << 56ULL) | r1; - return ret1 & ret2; - } + return ret1 & ret2; + } - // It's safe to read up to 64 bits. - // Preserves what's in result. - template <typename T> + // It's safe to read up to 64 bits. + // Preserves what's in result. + template <typename T> Y_FORCE_INLINE bool ReadSafe(T& result, ui64 bits, ui64 skipbits) { - ui64 r64 = 0; - bool ret = ReadSafe(r64, bits); - CopyToResult(result, r64, bits, skipbits); - return ret; - } - - // Do not try to read more than 56 bits at once. Split in two reads or use ReadSafe. - // Zeroes other bits in result. - template <typename T> + ui64 r64 = 0; + bool ret = ReadSafe(r64, bits); + CopyToResult(result, r64, bits, skipbits); + return ret; + } + + // Do not try to read more than 56 bits at once. Split in two reads or use ReadSafe. + // Zeroes other bits in result. + template <typename T> Y_FORCE_INLINE bool Read(T& result, ui64 bits) { - ui64 r64 = 0; - bool ret = ReadImpl(r64, bits); - result = r64; - return ret; - } + ui64 r64 = 0; + bool ret = ReadImpl(r64, bits); + result = r64; + return ret; + } - // Shortcut to impl. + // Shortcut to impl. Y_FORCE_INLINE bool Read(ui64& result, ui64 bits) { - return ReadImpl(result, bits); - } + return ReadImpl(result, bits); + } - // Do not try to read more than 56 bits at once. Split in two reads or use ReadSafe. - // Preserves what's in result. - template <typename T> + // Do not try to read more than 56 bits at once. Split in two reads or use ReadSafe. + // Preserves what's in result. + template <typename T> Y_FORCE_INLINE bool Read(T& result, ui64 bits, ui64 skipbits) { - ui64 r64 = 0; - bool ret = ReadImpl(r64, bits); - CopyToResult(result, r64, bits, skipbits); - return ret; - } - - // Unsigned wordwise read. Underlying data is splitted in "words" of "bits(data) + 1(flag)" bits. - // Like this: (unsigned char)0x2E<3> (0010 1110) <=> 1110 0101 - // fddd fddd - template <ui64 bits, typename T> + ui64 r64 = 0; + bool ret = ReadImpl(r64, bits); + CopyToResult(result, r64, bits, skipbits); + return ret; + } + + // Unsigned wordwise read. Underlying data is splitted in "words" of "bits(data) + 1(flag)" bits. + // Like this: (unsigned char)0x2E<3> (0010 1110) <=> 1110 0101 + // fddd fddd + template <ui64 bits, typename T> Y_FORCE_INLINE bool ReadWords(T& result) { - ui64 r64 = 0; + ui64 r64 = 0; - bool retCode = ReadWordsImpl<bits>(r64); - result = r64; + bool retCode = ReadWordsImpl<bits>(r64); + result = r64; - return retCode; - } + return retCode; + } - // Shortcut to impl. - template <ui64 bits> + // Shortcut to impl. + template <ui64 bits> Y_FORCE_INLINE bool ReadWords(ui64& result) { - return ReadWordsImpl<bits>(result); - } + return ReadWordsImpl<bits>(result); + } Y_FORCE_INLINE bool Back(int bits) { - return Seek(BitOffset() - bits); - } + return Seek(BitOffset() - bits); + } Y_FORCE_INLINE bool Seek(int bitoffset) { - return TBitInputImpl::Seek(bitoffset); - } + return TBitInputImpl::Seek(bitoffset); + } - // A way to read a portion of bits at random location. - // Didn't want to complicate sequential read, neither to copypaste. - template <typename T> + // A way to read a portion of bits at random location. + // Didn't want to complicate sequential read, neither to copypaste. + template <typename T> Y_FORCE_INLINE bool ReadRandom(ui64 bitoffset, T& result, ui64 bits, ui64 skipbits) { - const ui64 curr = BitOffset(); - Seek(bitoffset); - bool ret = ReadSafe<T>(result, bits, skipbits); - Seek(curr); - return ret; - } - }; + const ui64 curr = BitOffset(); + Seek(bitoffset); + bool ret = ReadSafe<T>(result, bits, skipbits); + Seek(curr); + return ret; + } + }; } diff --git a/library/cpp/bit_io/bitinput_impl.cpp b/library/cpp/bit_io/bitinput_impl.cpp index 81c897f882..e9ec98eeff 100644 --- a/library/cpp/bit_io/bitinput_impl.cpp +++ b/library/cpp/bit_io/bitinput_impl.cpp @@ -1 +1 @@ -#include "bitinput_impl.h" +#include "bitinput_impl.h" diff --git a/library/cpp/bit_io/bitinput_impl.h b/library/cpp/bit_io/bitinput_impl.h index b13fbef101..aa46b6e31b 100644 --- a/library/cpp/bit_io/bitinput_impl.h +++ b/library/cpp/bit_io/bitinput_impl.h @@ -1,110 +1,110 @@ #pragma once #include <util/generic/bitops.h> -#include <util/system/unaligned_mem.h> +#include <util/system/unaligned_mem.h> namespace NBitIO { - class TBitInputImpl { - i64 RealStart; - i64 Start; - ui64 Length; - ui64 BOffset; - const ui32 FakeStart; - char Fake[16]; - const i64 FStart; - - public: - TBitInputImpl(const char* start, const char* end) - : RealStart((i64)start) - , Start((i64)start) - , Length((end - start) << 3) - , BOffset(0) - , FakeStart(Length > 64 ? Length - 64 : 0) - , FStart((i64)Fake - (FakeStart >> 3)) - { - memcpy(Fake, (const char*)(RealStart + (FakeStart >> 3)), (Length - FakeStart) >> 3); - Start = FakeStart ? RealStart : FStart; - } + class TBitInputImpl { + i64 RealStart; + i64 Start; + ui64 Length; + ui64 BOffset; + const ui32 FakeStart; + char Fake[16]; + const i64 FStart; + + public: + TBitInputImpl(const char* start, const char* end) + : RealStart((i64)start) + , Start((i64)start) + , Length((end - start) << 3) + , BOffset(0) + , FakeStart(Length > 64 ? Length - 64 : 0) + , FStart((i64)Fake - (FakeStart >> 3)) + { + memcpy(Fake, (const char*)(RealStart + (FakeStart >> 3)), (Length - FakeStart) >> 3); + Start = FakeStart ? RealStart : FStart; + } ui64 GetBitLength() const { return Length; } - protected: - template <ui32 bits> + protected: + template <ui32 bits> Y_FORCE_INLINE bool ReadKImpl(ui64& result) { result = (ReadUnaligned<ui64>((const void*)(Start + (BOffset >> 3))) >> (BOffset & 7)) & Mask64(bits); - BOffset += bits; - if (BOffset < FakeStart) - return true; - if (BOffset > Length) { - result = 0; - BOffset -= bits; - return false; - } - Start = FStart; + BOffset += bits; + if (BOffset < FakeStart) + return true; + if (BOffset > Length) { + result = 0; + BOffset -= bits; + return false; + } + Start = FStart; return true; } Y_FORCE_INLINE bool ReadImpl(ui64& result, ui32 bits) { - result = (ReadUnaligned<ui64>((const void*)(Start + (BOffset >> 3))) >> (BOffset & 7)) & MaskLowerBits(bits); - BOffset += bits; - if (BOffset < FakeStart) - return true; + result = (ReadUnaligned<ui64>((const void*)(Start + (BOffset >> 3))) >> (BOffset & 7)) & MaskLowerBits(bits); + BOffset += bits; + if (BOffset < FakeStart) + return true; if (Y_UNLIKELY(BOffset > Length)) { - result = 0; - BOffset -= bits; - return false; - } - Start = FStart; + result = 0; + BOffset -= bits; + return false; + } + Start = FStart; return true; } Y_FORCE_INLINE bool EofImpl() const { - return BOffset >= Length; - } + return BOffset >= Length; + } Y_FORCE_INLINE ui64 BitOffset() const { - return BOffset; - } + return BOffset; + } Y_FORCE_INLINE bool Seek(i64 offset) { - if (offset < 0 || offset > (i64)Length) - return false; - BOffset = offset; - Start = BOffset < FakeStart ? RealStart : FStart; - return true; - } - - protected: - template <ui64 bits, typename T> + if (offset < 0 || offset > (i64)Length) + return false; + BOffset = offset; + Start = BOffset < FakeStart ? RealStart : FStart; + return true; + } + + protected: + template <ui64 bits, typename T> Y_FORCE_INLINE static void CopyToResultK(T& result, ui64 r64, ui64 skipbits) { result = (result & ~(Mask64(bits) << skipbits)) | (r64 << skipbits); - } + } - template <typename T> + template <typename T> Y_FORCE_INLINE static void CopyToResult(T& result, ui64 r64, ui64 bits, ui64 skipbits) { result = (result & InverseMaskLowerBits(bits, skipbits)) | (r64 << skipbits); - } + } - public: - template <ui64 bits> + public: + template <ui64 bits> Y_FORCE_INLINE bool ReadWordsImpl(ui64& data) { - data = 0; + data = 0; const ui64 haveMore = NthBit64(bits); const ui64 mask = Mask64(bits); - ui64 current = 0; - ui64 byteNo = 0; + ui64 current = 0; + ui64 byteNo = 0; - do { - if (!ReadKImpl<bits + 1>(current)) - return false; + do { + if (!ReadKImpl<bits + 1>(current)) + return false; - data |= (current & mask) << (byteNo++ * bits); - } while (current & haveMore); + data |= (current & mask) << (byteNo++ * bits); + } while (current & haveMore); - return true; - } - }; + return true; + } + }; } diff --git a/library/cpp/bit_io/bitoutput.cpp b/library/cpp/bit_io/bitoutput.cpp index d6c1c095da..a28265aa0b 100644 --- a/library/cpp/bit_io/bitoutput.cpp +++ b/library/cpp/bit_io/bitoutput.cpp @@ -1 +1 @@ -#include "bitoutput.h" +#include "bitoutput.h" diff --git a/library/cpp/bit_io/bitoutput.h b/library/cpp/bit_io/bitoutput.h index 2b886c1f02..0a6869b5d0 100644 --- a/library/cpp/bit_io/bitoutput.h +++ b/library/cpp/bit_io/bitoutput.h @@ -2,194 +2,194 @@ #include <library/cpp/deprecated/accessors/accessors.h> -#include <util/stream/output.h> -#include <util/system/yassert.h> +#include <util/stream/output.h> +#include <util/system/yassert.h> #include <util/generic/bitops.h> #include <util/generic/vector.h> #include <util/generic/yexception.h> namespace NBitIO { - // Based on junk/solar/codecs/bitstream.h + // Based on junk/solar/codecs/bitstream.h - // Almost all code is hard tuned for sequential write performance. - // Use tools/bursttrie/benchmarks/bitstreams_benchmark to check your changes. + // Almost all code is hard tuned for sequential write performance. + // Use tools/bursttrie/benchmarks/bitstreams_benchmark to check your changes. inline constexpr ui64 BytesUp(ui64 bits) { return (bits + 7ULL) >> 3ULL; } - template <typename TStorage> - class TBitOutputBase { - protected: - TStorage* Storage; - ui64 FreeBits; - ui64 Active; - ui64 Offset; - - public: - TBitOutputBase(TStorage* storage) - : Storage(storage) - , FreeBits(64) - , Active() - , Offset() - { - } - - ui64 GetOffset() const { + template <typename TStorage> + class TBitOutputBase { + protected: + TStorage* Storage; + ui64 FreeBits; + ui64 Active; + ui64 Offset; + + public: + TBitOutputBase(TStorage* storage) + : Storage(storage) + , FreeBits(64) + , Active() + , Offset() + { + } + + ui64 GetOffset() const { return Offset + BytesUp(64ULL - FreeBits); - } + } - ui64 GetBitOffset() const { - return (64ULL - FreeBits) & 7ULL; - } + ui64 GetBitOffset() const { + return (64ULL - FreeBits) & 7ULL; + } - ui64 GetByteReminder() const { - return FreeBits & 7ULL; - } + ui64 GetByteReminder() const { + return FreeBits & 7ULL; + } - public: - // interface + public: + // interface - // Write "bits" lower bits. + // Write "bits" lower bits. Y_FORCE_INLINE void Write(ui64 data, ui64 bits) { - if (FreeBits < bits) { - if (FreeBits) { - bits -= FreeBits; - - Active |= (data & MaskLowerBits(FreeBits)) << (64ULL - FreeBits); - data >>= FreeBits; - - FreeBits = 0ULL; - } - - Flush(); - } - + if (FreeBits < bits) { + if (FreeBits) { + bits -= FreeBits; + + Active |= (data & MaskLowerBits(FreeBits)) << (64ULL - FreeBits); + data >>= FreeBits; + + FreeBits = 0ULL; + } + + Flush(); + } + Active |= bits ? ((data & MaskLowerBits(bits)) << (64ULL - FreeBits)) : 0; - FreeBits -= bits; + FreeBits -= bits; } - // Write "bits" lower bits starting from "skipbits" bit. + // Write "bits" lower bits starting from "skipbits" bit. Y_FORCE_INLINE void Write(ui64 data, ui64 bits, ui64 skipbits) { - Write(data >> skipbits, bits); - } + Write(data >> skipbits, bits); + } - // Unsigned wordwise write. Underlying data is splitted in "words" of "bits(data) + 1(flag)" bits. - // Like this: (unsigned char)0x2E<3> (0000 0010 1110) <=> 1110 0101 - // fddd fddd - template <ui64 bits> + // Unsigned wordwise write. Underlying data is splitted in "words" of "bits(data) + 1(flag)" bits. + // Like this: (unsigned char)0x2E<3> (0000 0010 1110) <=> 1110 0101 + // fddd fddd + template <ui64 bits> Y_FORCE_INLINE void WriteWords(ui64 data) { - do { - ui64 part = data; + do { + ui64 part = data; - data >>= bits; + data >>= bits; part |= FastZeroIfFalse(data, NthBit64(bits)); - Write(part, bits + 1ULL); - } while (data); - } + Write(part, bits + 1ULL); + } while (data); + } Y_FORCE_INLINE ui64 /* padded bits */ Flush() { - const ui64 ubytes = 8ULL - (FreeBits >> 3ULL); + const ui64 ubytes = 8ULL - (FreeBits >> 3ULL); - if (ubytes) { - Active <<= FreeBits; - Active >>= FreeBits; + if (ubytes) { + Active <<= FreeBits; + Active >>= FreeBits; - Storage->WriteData((const char*)&Active, (const char*)&Active + ubytes); - Offset += ubytes; - } + Storage->WriteData((const char*)&Active, (const char*)&Active + ubytes); + Offset += ubytes; + } - const ui64 padded = FreeBits & 7; + const ui64 padded = FreeBits & 7; - FreeBits = 64ULL; - Active = 0ULL; + FreeBits = 64ULL; + Active = 0ULL; - return padded; - } + return padded; + } - virtual ~TBitOutputBase() { - Flush(); - } + virtual ~TBitOutputBase() { + Flush(); + } private: static Y_FORCE_INLINE ui64 FastZeroIfFalse(bool cond, ui64 iftrue) { return -i64(cond) & iftrue; } - }; + }; - template <typename TVec> - class TBitOutputVectorImpl { - TVec* Data; + template <typename TVec> + class TBitOutputVectorImpl { + TVec* Data; - public: - void WriteData(const char* begin, const char* end) { + public: + void WriteData(const char* begin, const char* end) { NAccessors::Append(*Data, begin, end); - } - - TBitOutputVectorImpl(TVec* data) - : Data(data) - { - } - }; - - template <typename TVec> - struct TBitOutputVector: public TBitOutputVectorImpl<TVec>, public TBitOutputBase<TBitOutputVectorImpl<TVec>> { - inline TBitOutputVector(TVec* data) - : TBitOutputVectorImpl<TVec>(data) - , TBitOutputBase<TBitOutputVectorImpl<TVec>>(this) - { - } - }; - - class TBitOutputArrayImpl { - char* Data; - size_t Left; - - public: - void WriteData(const char* begin, const char* end) { - size_t sz = end - begin; + } + + TBitOutputVectorImpl(TVec* data) + : Data(data) + { + } + }; + + template <typename TVec> + struct TBitOutputVector: public TBitOutputVectorImpl<TVec>, public TBitOutputBase<TBitOutputVectorImpl<TVec>> { + inline TBitOutputVector(TVec* data) + : TBitOutputVectorImpl<TVec>(data) + , TBitOutputBase<TBitOutputVectorImpl<TVec>>(this) + { + } + }; + + class TBitOutputArrayImpl { + char* Data; + size_t Left; + + public: + void WriteData(const char* begin, const char* end) { + size_t sz = end - begin; Y_VERIFY(sz <= Left, " "); - memcpy(Data, begin, sz); - Data += sz; - Left -= sz; - } - - TBitOutputArrayImpl(char* begin, size_t len) - : Data(begin) - , Left(len) - { - } - }; - - struct TBitOutputArray: public TBitOutputArrayImpl, public TBitOutputBase<TBitOutputArrayImpl> { - inline TBitOutputArray(char* begin, size_t len) - : TBitOutputArrayImpl(begin, len) - , TBitOutputBase<TBitOutputArrayImpl>(this) - { - } - }; - + memcpy(Data, begin, sz); + Data += sz; + Left -= sz; + } + + TBitOutputArrayImpl(char* begin, size_t len) + : Data(begin) + , Left(len) + { + } + }; + + struct TBitOutputArray: public TBitOutputArrayImpl, public TBitOutputBase<TBitOutputArrayImpl> { + inline TBitOutputArray(char* begin, size_t len) + : TBitOutputArrayImpl(begin, len) + , TBitOutputBase<TBitOutputArrayImpl>(this) + { + } + }; + using TBitOutputYVector = TBitOutputVector<TVector<char>>; - class TBitOutputStreamImpl { + class TBitOutputStreamImpl { IOutputStream* Out; - public: - void WriteData(const char* begin, const char* end) { - Out->Write(begin, end - begin); - } + public: + void WriteData(const char* begin, const char* end) { + Out->Write(begin, end - begin); + } TBitOutputStreamImpl(IOutputStream* out) - : Out(out) - { - } - }; - - struct TBitOutputStream: public TBitOutputStreamImpl, public TBitOutputBase<TBitOutputStreamImpl> { + : Out(out) + { + } + }; + + struct TBitOutputStream: public TBitOutputStreamImpl, public TBitOutputBase<TBitOutputStreamImpl> { inline TBitOutputStream(IOutputStream* out) - : TBitOutputStreamImpl(out) - , TBitOutputBase<TBitOutputStreamImpl>(this) - { - } - }; + : TBitOutputStreamImpl(out) + , TBitOutputBase<TBitOutputStreamImpl>(this) + { + } + }; } diff --git a/library/cpp/bit_io/ut/ya.make b/library/cpp/bit_io/ut/ya.make index 07ee5b4997..758fccad62 100644 --- a/library/cpp/bit_io/ut/ya.make +++ b/library/cpp/bit_io/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/bit_io) - -OWNER( + +OWNER( velavokr g:util -) - -SRCS( - bitinout_ut.cpp -) - -END() +) + +SRCS( + bitinout_ut.cpp +) + +END() diff --git a/library/cpp/bit_io/ya.make b/library/cpp/bit_io/ya.make index df1de81ff9..cfc83f145e 100644 --- a/library/cpp/bit_io/ya.make +++ b/library/cpp/bit_io/ya.make @@ -1,18 +1,18 @@ -LIBRARY() - -OWNER( - velavokr +LIBRARY() + +OWNER( + velavokr g:util -) - -PEERDIR( +) + +PEERDIR( library/cpp/deprecated/accessors -) - -SRCS( - bitinput.cpp - bitinput_impl.cpp - bitoutput.cpp -) - -END() +) + +SRCS( + bitinput.cpp + bitinput_impl.cpp + bitoutput.cpp +) + +END() diff --git a/library/cpp/blockcodecs/codecs/legacy_zstd06/legacy_zstd06.cpp b/library/cpp/blockcodecs/codecs/legacy_zstd06/legacy_zstd06.cpp index 042f031679..7319008420 100644 --- a/library/cpp/blockcodecs/codecs/legacy_zstd06/legacy_zstd06.cpp +++ b/library/cpp/blockcodecs/codecs/legacy_zstd06/legacy_zstd06.cpp @@ -1,58 +1,58 @@ #include <library/cpp/blockcodecs/core/codecs.h> #include <library/cpp/blockcodecs/core/common.h> #include <library/cpp/blockcodecs/core/register.h> - + #include <contrib/libs/zstd06/common/zstd.h> #include <contrib/libs/zstd06/common/zstd_static.h> - -using namespace NBlockCodecs; - -namespace { + +using namespace NBlockCodecs; + +namespace { struct TZStd06Codec: public TAddLengthCodec<TZStd06Codec> { inline TZStd06Codec(unsigned level) - : Level(level) + : Level(level) , MyName(TStringBuf("zstd06_") + ToString(Level)) - { - } - - static inline size_t CheckError(size_t ret, const char* what) { - if (ZSTD_isError(ret)) { + { + } + + static inline size_t CheckError(size_t ret, const char* what) { + if (ZSTD_isError(ret)) { ythrow yexception() << what << TStringBuf(" zstd error: ") << ZSTD_getErrorName(ret); - } - - return ret; - } - + } + + return ret; + } + static inline size_t DoMaxCompressedLength(size_t l) noexcept { - return ZSTD_compressBound(l); - } - - inline size_t DoCompress(const TData& in, void* out) const { + return ZSTD_compressBound(l); + } + + inline size_t DoCompress(const TData& in, void* out) const { return CheckError(ZSTD_compress(out, DoMaxCompressedLength(in.size()), in.data(), in.size(), Level), "compress"); - } - - inline void DoDecompress(const TData& in, void* out, size_t dsize) const { + } + + inline void DoDecompress(const TData& in, void* out, size_t dsize) const { const size_t res = CheckError(ZSTD_decompress(out, dsize, in.data(), in.size()), "decompress"); - - if (res != dsize) { - ythrow TDecompressError(dsize, res); - } - } - + + if (res != dsize) { + ythrow TDecompressError(dsize, res); + } + } + TStringBuf Name() const noexcept override { - return MyName; - } - - const unsigned Level; + return MyName; + } + + const unsigned Level; const TString MyName; - }; - + }; + struct TZStd06Registrar { TZStd06Registrar() { for (unsigned i = 1; i <= ZSTD_maxCLevel(); ++i) { RegisterCodec(MakeHolder<TZStd06Codec>(i)); } - } + } }; const TZStd06Registrar Registrar{}; -} +} diff --git a/library/cpp/blockcodecs/codecs_ut.cpp b/library/cpp/blockcodecs/codecs_ut.cpp index bfe5a23690..62acd66f9f 100644 --- a/library/cpp/blockcodecs/codecs_ut.cpp +++ b/library/cpp/blockcodecs/codecs_ut.cpp @@ -1,24 +1,24 @@ -#include "codecs.h" -#include "stream.h" - +#include "codecs.h" +#include "stream.h" + #include <library/cpp/testing/unittest/registar.h> - -#include <util/stream/str.h> + +#include <util/stream/str.h> #include <util/string/join.h> -#include <util/digest/multi.h> - +#include <util/digest/multi.h> + Y_UNIT_TEST_SUITE(TBlockCodecsTest) { - using namespace NBlockCodecs; - + using namespace NBlockCodecs; + TBuffer Buffer(TStringBuf b) { TBuffer bb; bb.Assign(b.data(), b.size()); return bb; } - void TestAllAtOnce(size_t n, size_t m) { + void TestAllAtOnce(size_t n, size_t m) { TVector<TBuffer> datas; - + datas.emplace_back(); datas.push_back(Buffer("na gorshke sidel korol")); datas.push_back(Buffer(TStringBuf("", 1))); @@ -26,249 +26,249 @@ Y_UNIT_TEST_SUITE(TBlockCodecsTest) { datas.push_back(Buffer(" ")); datas.push_back(Buffer(" ")); datas.push_back(Buffer(" ")); - - { - TStringStream data; - - for (size_t i = 0; i < 1024; ++i) { - data << " " << i; - } - - datas.push_back(Buffer(data.Str())); - } - - TCodecList lst = ListAllCodecs(); - + + { + TStringStream data; + + for (size_t i = 0; i < 1024; ++i) { + data << " " << i; + } + + datas.push_back(Buffer(data.Str())); + } + + TCodecList lst = ListAllCodecs(); + for (size_t i = 0; i < lst.size(); ++i) { - const ICodec* c = Codec(lst[i]); - const auto h = MultiHash(c->Name(), i, 1); - - if (h % n == m) { - } else { - continue; - } - + const ICodec* c = Codec(lst[i]); + const auto h = MultiHash(c->Name(), i, 1); + + if (h % n == m) { + } else { + continue; + } + for (size_t j = 0; j < datas.size(); ++j) { const TBuffer& data = datas[j]; TString res; - try { + try { TBuffer e, d; c->Encode(data, e); c->Decode(e, d); d.AsString(res); UNIT_ASSERT_EQUAL(NBlockCodecs::TData(res), NBlockCodecs::TData(data)); - } catch (...) { + } catch (...) { Cerr << c->Name() << "(" << res.Quote() << ")(" << TString{NBlockCodecs::TData(data)}.Quote() << ")" << Endl; - - throw; - } - } - } - } - + + throw; + } + } + } + } + Y_UNIT_TEST(TestAllAtOnce0) { - TestAllAtOnce(20, 0); - } - + TestAllAtOnce(20, 0); + } + Y_UNIT_TEST(TestAllAtOnce1) { - TestAllAtOnce(20, 1); - } - + TestAllAtOnce(20, 1); + } + Y_UNIT_TEST(TestAllAtOnce2) { - TestAllAtOnce(20, 2); - } - + TestAllAtOnce(20, 2); + } + Y_UNIT_TEST(TestAllAtOnce3) { - TestAllAtOnce(20, 3); - } - + TestAllAtOnce(20, 3); + } + Y_UNIT_TEST(TestAllAtOnce4) { - TestAllAtOnce(20, 4); - } - + TestAllAtOnce(20, 4); + } + Y_UNIT_TEST(TestAllAtOnce5) { - TestAllAtOnce(20, 5); - } - + TestAllAtOnce(20, 5); + } + Y_UNIT_TEST(TestAllAtOnce6) { - TestAllAtOnce(20, 6); - } - + TestAllAtOnce(20, 6); + } + Y_UNIT_TEST(TestAllAtOnce7) { - TestAllAtOnce(20, 7); - } - + TestAllAtOnce(20, 7); + } + Y_UNIT_TEST(TestAllAtOnce8) { - TestAllAtOnce(20, 8); - } - + TestAllAtOnce(20, 8); + } + Y_UNIT_TEST(TestAllAtOnce9) { - TestAllAtOnce(20, 9); - } - + TestAllAtOnce(20, 9); + } + Y_UNIT_TEST(TestAllAtOnce10) { - TestAllAtOnce(20, 10); - } - + TestAllAtOnce(20, 10); + } + Y_UNIT_TEST(TestAllAtOnce12) { - TestAllAtOnce(20, 12); - } - + TestAllAtOnce(20, 12); + } + Y_UNIT_TEST(TestAllAtOnce13) { - TestAllAtOnce(20, 13); - } - + TestAllAtOnce(20, 13); + } + Y_UNIT_TEST(TestAllAtOnce14) { - TestAllAtOnce(20, 14); - } - + TestAllAtOnce(20, 14); + } + Y_UNIT_TEST(TestAllAtOnce15) { - TestAllAtOnce(20, 15); - } - + TestAllAtOnce(20, 15); + } + Y_UNIT_TEST(TestAllAtOnce16) { - TestAllAtOnce(20, 16); - } - + TestAllAtOnce(20, 16); + } + Y_UNIT_TEST(TestAllAtOnce17) { - TestAllAtOnce(20, 17); - } - + TestAllAtOnce(20, 17); + } + Y_UNIT_TEST(TestAllAtOnce18) { - TestAllAtOnce(20, 18); - } - + TestAllAtOnce(20, 18); + } + Y_UNIT_TEST(TestAllAtOnce19) { - TestAllAtOnce(20, 19); - } - - void TestStreams(size_t n, size_t m) { + TestAllAtOnce(20, 19); + } + + void TestStreams(size_t n, size_t m) { TVector<TString> datas; TString res; - - for (size_t i = 0; i < 256; ++i) { + + for (size_t i = 0; i < 256; ++i) { datas.push_back(TString(i, (char)(i % 128))); - } - + } + for (size_t i = 0; i < datas.size(); ++i) { - res += datas[i]; - } - - TCodecList lst = ListAllCodecs(); - + res += datas[i]; + } + + TCodecList lst = ListAllCodecs(); + for (size_t i = 0; i < lst.size(); ++i) { - TStringStream ss; - - const ICodec* c = Codec(lst[i]); - const auto h = MultiHash(c->Name(), i, 2); - - if (h % n == m) { - } else { - continue; - } - - { - TCodedOutput out(&ss, c, 1234); - + TStringStream ss; + + const ICodec* c = Codec(lst[i]); + const auto h = MultiHash(c->Name(), i, 2); + + if (h % n == m) { + } else { + continue; + } + + { + TCodedOutput out(&ss, c, 1234); + for (size_t j = 0; j < datas.size(); ++j) { - out << datas[j]; - } - - out.Finish(); - } - + out << datas[j]; + } + + out.Finish(); + } + const TString resNew = TDecodedInput(&ss).ReadAll(); - - try { - UNIT_ASSERT_EQUAL(resNew, res); - } catch (...) { - Cerr << c->Name() << Endl; - - throw; - } - } - } - + + try { + UNIT_ASSERT_EQUAL(resNew, res); + } catch (...) { + Cerr << c->Name() << Endl; + + throw; + } + } + } + Y_UNIT_TEST(TestStreams0) { - TestStreams(20, 0); - } - + TestStreams(20, 0); + } + Y_UNIT_TEST(TestStreams1) { - TestStreams(20, 1); - } - + TestStreams(20, 1); + } + Y_UNIT_TEST(TestStreams2) { - TestStreams(20, 2); - } - + TestStreams(20, 2); + } + Y_UNIT_TEST(TestStreams3) { - TestStreams(20, 3); - } - + TestStreams(20, 3); + } + Y_UNIT_TEST(TestStreams4) { - TestStreams(20, 4); - } - + TestStreams(20, 4); + } + Y_UNIT_TEST(TestStreams5) { - TestStreams(20, 5); - } - + TestStreams(20, 5); + } + Y_UNIT_TEST(TestStreams6) { - TestStreams(20, 6); - } - + TestStreams(20, 6); + } + Y_UNIT_TEST(TestStreams7) { - TestStreams(20, 7); - } - + TestStreams(20, 7); + } + Y_UNIT_TEST(TestStreams8) { - TestStreams(20, 8); - } - + TestStreams(20, 8); + } + Y_UNIT_TEST(TestStreams9) { - TestStreams(20, 9); - } - + TestStreams(20, 9); + } + Y_UNIT_TEST(TestStreams10) { - TestStreams(20, 10); - } - + TestStreams(20, 10); + } + Y_UNIT_TEST(TestStreams11) { - TestStreams(20, 11); - } - + TestStreams(20, 11); + } + Y_UNIT_TEST(TestStreams12) { - TestStreams(20, 12); - } - + TestStreams(20, 12); + } + Y_UNIT_TEST(TestStreams13) { - TestStreams(20, 13); - } - + TestStreams(20, 13); + } + Y_UNIT_TEST(TestStreams14) { - TestStreams(20, 14); - } - + TestStreams(20, 14); + } + Y_UNIT_TEST(TestStreams15) { - TestStreams(20, 15); - } - + TestStreams(20, 15); + } + Y_UNIT_TEST(TestStreams16) { - TestStreams(20, 16); - } - + TestStreams(20, 16); + } + Y_UNIT_TEST(TestStreams17) { - TestStreams(20, 17); - } - + TestStreams(20, 17); + } + Y_UNIT_TEST(TestStreams18) { - TestStreams(20, 18); - } - + TestStreams(20, 18); + } + Y_UNIT_TEST(TestStreams19) { - TestStreams(20, 19); - } + TestStreams(20, 19); + } Y_UNIT_TEST(TestMaxPossibleDecompressedSize) { @@ -337,4 +337,4 @@ Y_UNIT_TEST_SUITE(TBlockCodecsTest) { UNIT_ASSERT_VALUES_EQUAL(decoded, data); } } -} +} diff --git a/library/cpp/blockcodecs/core/codecs.cpp b/library/cpp/blockcodecs/core/codecs.cpp index 21506e812b..4f6b15b550 100644 --- a/library/cpp/blockcodecs/core/codecs.cpp +++ b/library/cpp/blockcodecs/core/codecs.cpp @@ -1,91 +1,91 @@ -#include "codecs.h" +#include "codecs.h" #include "common.h" #include "register.h" - -#include <util/ysaveload.h> + +#include <util/ysaveload.h> #include <util/stream/null.h> -#include <util/stream/mem.h> -#include <util/string/cast.h> -#include <util/string/join.h> -#include <util/system/align.h> -#include <util/system/unaligned_mem.h> -#include <util/generic/hash.h> -#include <util/generic/cast.h> +#include <util/stream/mem.h> +#include <util/string/cast.h> +#include <util/string/join.h> +#include <util/system/align.h> +#include <util/system/unaligned_mem.h> +#include <util/generic/hash.h> +#include <util/generic/cast.h> #include <util/generic/deque.h> -#include <util/generic/buffer.h> +#include <util/generic/buffer.h> #include <util/generic/array_ref.h> -#include <util/generic/singleton.h> -#include <util/generic/algorithm.h> -#include <util/generic/mem_copy.h> - -using namespace NBlockCodecs; - -namespace { - - struct TCodecFactory { - inline TCodecFactory() { - Add(&Null); - } - - inline const ICodec* Find(const TStringBuf& name) const { - auto it = Registry.find(name); - - if (it == Registry.end()) { - ythrow TNotFound() << "can not found " << name << " codec"; - } - - return it->second; - } - - inline void ListCodecs(TCodecList& lst) const { +#include <util/generic/singleton.h> +#include <util/generic/algorithm.h> +#include <util/generic/mem_copy.h> + +using namespace NBlockCodecs; + +namespace { + + struct TCodecFactory { + inline TCodecFactory() { + Add(&Null); + } + + inline const ICodec* Find(const TStringBuf& name) const { + auto it = Registry.find(name); + + if (it == Registry.end()) { + ythrow TNotFound() << "can not found " << name << " codec"; + } + + return it->second; + } + + inline void ListCodecs(TCodecList& lst) const { for (const auto& it : Registry) { lst.push_back(it.first); - } - - Sort(lst.begin(), lst.end()); - } - - inline void Add(ICodec* codec) { - Registry[codec->Name()] = codec; - } - + } + + Sort(lst.begin(), lst.end()); + } + + inline void Add(ICodec* codec) { + Registry[codec->Name()] = codec; + } + inline void Add(TCodecPtr codec) { Codecs.push_back(std::move(codec)); Add(Codecs.back().Get()); } - inline void Alias(TStringBuf from, TStringBuf to) { - Tmp.emplace_back(from); - Registry[Tmp.back()] = Registry[to]; - } - + inline void Alias(TStringBuf from, TStringBuf to) { + Tmp.emplace_back(from); + Registry[Tmp.back()] = Registry[to]; + } + TDeque<TString> Tmp; - TNullCodec Null; + TNullCodec Null; TVector<TCodecPtr> Codecs; typedef THashMap<TStringBuf, ICodec*> TRegistry; - TRegistry Registry; + TRegistry Registry; // SEARCH-8344: Global decompressed size limiter (to prevent remote DoS) size_t MaxPossibleDecompressedLength = Max<size_t>(); - }; -} - -const ICodec* NBlockCodecs::Codec(const TStringBuf& name) { - return Singleton<TCodecFactory>()->Find(name); -} - -TCodecList NBlockCodecs::ListAllCodecs() { - TCodecList ret; - - Singleton<TCodecFactory>()->ListCodecs(ret); - - return ret; -} - + }; +} + +const ICodec* NBlockCodecs::Codec(const TStringBuf& name) { + return Singleton<TCodecFactory>()->Find(name); +} + +TCodecList NBlockCodecs::ListAllCodecs() { + TCodecList ret; + + Singleton<TCodecFactory>()->ListCodecs(ret); + + return ret; +} + TString NBlockCodecs::ListAllCodecsAsString() { return JoinSeq(TStringBuf(","), ListAllCodecs()); -} - +} + void NBlockCodecs::RegisterCodec(TCodecPtr codec) { Singleton<TCodecFactory>()->Add(std::move(codec)); } @@ -113,36 +113,36 @@ size_t ICodec::GetDecompressedLength(const TData& in) const { return len; } -void ICodec::Encode(const TData& in, TBuffer& out) const { - const size_t maxLen = MaxCompressedLength(in); - - out.Reserve(maxLen); - out.Resize(Compress(in, out.Data())); -} - -void ICodec::Decode(const TData& in, TBuffer& out) const { +void ICodec::Encode(const TData& in, TBuffer& out) const { + const size_t maxLen = MaxCompressedLength(in); + + out.Reserve(maxLen); + out.Resize(Compress(in, out.Data())); +} + +void ICodec::Decode(const TData& in, TBuffer& out) const { const size_t len = GetDecompressedLength(in); - - out.Reserve(len); - out.Resize(Decompress(in, out.Data())); -} - + + out.Reserve(len); + out.Resize(Decompress(in, out.Data())); +} + void ICodec::Encode(const TData& in, TString& out) const { - const size_t maxLen = MaxCompressedLength(in); + const size_t maxLen = MaxCompressedLength(in); out.ReserveAndResize(maxLen); - + size_t actualLen = Compress(in, out.begin()); Y_ASSERT(actualLen <= maxLen); out.resize(actualLen); -} - +} + void ICodec::Decode(const TData& in, TString& out) const { const size_t maxLen = GetDecompressedLength(in); out.ReserveAndResize(maxLen); - + size_t actualLen = Decompress(in, out.begin()); Y_ASSERT(actualLen <= maxLen); out.resize(actualLen); -} - +} + ICodec::~ICodec() = default; diff --git a/library/cpp/blockcodecs/core/codecs.h b/library/cpp/blockcodecs/core/codecs.h index 9c93c00274..90b2e1842c 100644 --- a/library/cpp/blockcodecs/core/codecs.h +++ b/library/cpp/blockcodecs/core/codecs.h @@ -1,14 +1,14 @@ -#pragma once - -#include <util/generic/buffer.h> -#include <util/generic/strbuf.h> +#pragma once + +#include <util/generic/buffer.h> +#include <util/generic/strbuf.h> #include <util/generic/string.h> #include <util/generic/typetraits.h> -#include <util/generic/vector.h> -#include <util/generic/yexception.h> - -namespace NBlockCodecs { - struct TData: public TStringBuf { +#include <util/generic/vector.h> +#include <util/generic/yexception.h> + +namespace NBlockCodecs { + struct TData: public TStringBuf { inline TData() = default; Y_HAS_MEMBER(Data); @@ -18,68 +18,68 @@ namespace NBlockCodecs { inline TData(const T& t) : TStringBuf((const char*)t.data(), t.size()) { - } - + } + template <class T, std::enable_if_t<THasSize<T>::value && THasData<T>::value, int> = 0> - inline TData(const T& t) + inline TData(const T& t) : TStringBuf((const char*)t.Data(), t.Size()) - { - } - }; - - struct TCodecError: public yexception { - }; - - struct TNotFound: public TCodecError { - }; - - struct TDataError: public TCodecError { - }; - - struct ICodec { - virtual ~ICodec(); - + { + } + }; + + struct TCodecError: public yexception { + }; + + struct TNotFound: public TCodecError { + }; + + struct TDataError: public TCodecError { + }; + + struct ICodec { + virtual ~ICodec(); + // main interface - virtual size_t DecompressedLength(const TData& in) const = 0; - virtual size_t MaxCompressedLength(const TData& in) const = 0; - virtual size_t Compress(const TData& in, void* out) const = 0; - virtual size_t Decompress(const TData& in, void* out) const = 0; - + virtual size_t DecompressedLength(const TData& in) const = 0; + virtual size_t MaxCompressedLength(const TData& in) const = 0; + virtual size_t Compress(const TData& in, void* out) const = 0; + virtual size_t Decompress(const TData& in, void* out) const = 0; + virtual TStringBuf Name() const noexcept = 0; - + // some useful helpers - void Encode(const TData& in, TBuffer& out) const; - void Decode(const TData& in, TBuffer& out) const; - + void Encode(const TData& in, TBuffer& out) const; + void Decode(const TData& in, TBuffer& out) const; + void Encode(const TData& in, TString& out) const; void Decode(const TData& in, TString& out) const; - + inline TString Encode(const TData& in) const { TString out; - - Encode(in, out); - - return out; - } - + + Encode(in, out); + + return out; + } + inline TString Decode(const TData& in) const { TString out; - - Decode(in, out); - - return out; - } + + Decode(in, out); + + return out; + } private: size_t GetDecompressedLength(const TData& in) const; - }; - + }; + using TCodecPtr = THolder<ICodec>; - const ICodec* Codec(const TStringBuf& name); - + const ICodec* Codec(const TStringBuf& name); + // some aux methods typedef TVector<TStringBuf> TCodecList; - TCodecList ListAllCodecs(); + TCodecList ListAllCodecs(); TString ListAllCodecsAsString(); // SEARCH-8344: Get the size of max possible decompressed block @@ -87,4 +87,4 @@ namespace NBlockCodecs { // SEARCH-8344: Globally set the size of max possible decompressed block void SetMaxPossibleDecompressedLength(size_t maxPossibleDecompressedLength); -} +} diff --git a/library/cpp/blockcodecs/core/common.h b/library/cpp/blockcodecs/core/common.h index f05df4d334..33649cf567 100644 --- a/library/cpp/blockcodecs/core/common.h +++ b/library/cpp/blockcodecs/core/common.h @@ -1,105 +1,105 @@ #pragma once - -#include "codecs.h" - -#include <util/ysaveload.h> + +#include "codecs.h" + +#include <util/ysaveload.h> #include <util/stream/null.h> -#include <util/stream/mem.h> -#include <util/string/cast.h> -#include <util/string/join.h> -#include <util/system/align.h> -#include <util/system/unaligned_mem.h> -#include <util/generic/hash.h> -#include <util/generic/cast.h> -#include <util/generic/buffer.h> +#include <util/stream/mem.h> +#include <util/string/cast.h> +#include <util/string/join.h> +#include <util/system/align.h> +#include <util/system/unaligned_mem.h> +#include <util/generic/hash.h> +#include <util/generic/cast.h> +#include <util/generic/buffer.h> #include <util/generic/array_ref.h> -#include <util/generic/singleton.h> -#include <util/generic/algorithm.h> -#include <util/generic/mem_copy.h> - +#include <util/generic/singleton.h> +#include <util/generic/algorithm.h> +#include <util/generic/mem_copy.h> + namespace NBlockCodecs { - struct TDecompressError: public TDataError { - TDecompressError(int code) { - *this << "cannot decompress (errcode " << code << ")"; - } - - TDecompressError(size_t exp, size_t real) { - *this << "broken input (expected len: " << exp << ", got: " << real << ")"; - } + struct TDecompressError: public TDataError { + TDecompressError(int code) { + *this << "cannot decompress (errcode " << code << ")"; + } + + TDecompressError(size_t exp, size_t real) { + *this << "broken input (expected len: " << exp << ", got: " << real << ")"; + } }; - struct TCompressError: public TDataError { - TCompressError(int code) { - *this << "cannot compress (errcode " << code << ")"; - } + struct TCompressError: public TDataError { + TCompressError(int code) { + *this << "cannot compress (errcode " << code << ")"; + } }; - struct TNullCodec: public ICodec { + struct TNullCodec: public ICodec { size_t DecompressedLength(const TData& in) const override { return in.size(); - } - + } + size_t MaxCompressedLength(const TData& in) const override { return in.size(); - } - + } + size_t Compress(const TData& in, void* out) const override { MemCopy((char*)out, in.data(), in.size()); - + return in.size(); - } - + } + size_t Decompress(const TData& in, void* out) const override { MemCopy((char*)out, in.data(), in.size()); - + return in.size(); - } - + } + TStringBuf Name() const noexcept override { return TStringBuf("null"); - } - }; - - template <class T> - struct TAddLengthCodec: public ICodec { - static inline void Check(const TData& in) { + } + }; + + template <class T> + struct TAddLengthCodec: public ICodec { + static inline void Check(const TData& in) { if (in.size() < sizeof(ui64)) { - ythrow TDataError() << "too small input"; - } - } - + ythrow TDataError() << "too small input"; + } + } + size_t DecompressedLength(const TData& in) const override { - Check(in); - + Check(in); + return ReadUnaligned<ui64>(in.data()); - } - + } + size_t MaxCompressedLength(const TData& in) const override { return T::DoMaxCompressedLength(in.size()) + sizeof(ui64); - } - + } + size_t Compress(const TData& in, void* out) const override { - ui64* ptr = (ui64*)out; - + ui64* ptr = (ui64*)out; + WriteUnaligned<ui64>(ptr, (ui64) in.size()); - + return Base()->DoCompress(!in ? TData(TStringBuf("")) : in, ptr + 1) + sizeof(*ptr); - } - + } + size_t Decompress(const TData& in, void* out) const override { - Check(in); - + Check(in); + const auto len = ReadUnaligned<ui64>(in.data()); - + if (!len) return 0; Base()->DoDecompress(TData(in).Skip(sizeof(len)), out, len); return len; - } - + } + inline const T* Base() const noexcept { - return static_cast<const T*>(this); - } - }; -} + return static_cast<const T*>(this); + } + }; +} diff --git a/library/cpp/blockcodecs/core/stream.cpp b/library/cpp/blockcodecs/core/stream.cpp index 4f7db3c32b..c0134dea28 100644 --- a/library/cpp/blockcodecs/core/stream.cpp +++ b/library/cpp/blockcodecs/core/stream.cpp @@ -1,212 +1,212 @@ -#include "stream.h" -#include "codecs.h" - -#include <util/digest/murmur.h> -#include <util/generic/scope.h> -#include <util/generic/cast.h> -#include <util/generic/hash.h> -#include <util/generic/singleton.h> -#include <util/stream/mem.h> -#include <util/ysaveload.h> - -using namespace NBlockCodecs; - -namespace { +#include "stream.h" +#include "codecs.h" + +#include <util/digest/murmur.h> +#include <util/generic/scope.h> +#include <util/generic/cast.h> +#include <util/generic/hash.h> +#include <util/generic/singleton.h> +#include <util/stream/mem.h> +#include <util/ysaveload.h> + +using namespace NBlockCodecs; + +namespace { constexpr size_t MAX_BUF_LEN = 128 * 1024 * 1024; - - typedef ui16 TCodecID; - typedef ui64 TBlockLen; - - struct TIds { - inline TIds() { - const TCodecList lst = ListAllCodecs(); - + + typedef ui16 TCodecID; + typedef ui64 TBlockLen; + + struct TIds { + inline TIds() { + const TCodecList lst = ListAllCodecs(); + for (size_t i = 0; i < lst.size(); ++i) { - const ICodec* c = Codec(lst[i]); - - ByID[CodecID(c)] = c; - } - } - - static inline TCodecID CodecID(const ICodec* c) { - const TStringBuf name = c->Name(); - - union { - ui16 Parts[2]; - ui32 Data; - } x; - + const ICodec* c = Codec(lst[i]); + + ByID[CodecID(c)] = c; + } + } + + static inline TCodecID CodecID(const ICodec* c) { + const TStringBuf name = c->Name(); + + union { + ui16 Parts[2]; + ui32 Data; + } x; + x.Data = MurmurHash<ui32>(name.data(), name.size()); - - return x.Parts[1] ^ x.Parts[0]; - } - - inline const ICodec* Find(TCodecID id) const { - TByID::const_iterator it = ByID.find(id); - - if (it != ByID.end()) { - return it->second; - } - - ythrow yexception() << "can not find codec by id " << id; - } - + + return x.Parts[1] ^ x.Parts[0]; + } + + inline const ICodec* Find(TCodecID id) const { + TByID::const_iterator it = ByID.find(id); + + if (it != ByID.end()) { + return it->second; + } + + ythrow yexception() << "can not find codec by id " << id; + } + typedef THashMap<TCodecID, const ICodec*> TByID; - TByID ByID; - }; - + TByID ByID; + }; + TCodecID CodecID(const ICodec* c) { - return TIds::CodecID(c); - } - + return TIds::CodecID(c); + } + const ICodec* CodecByID(TCodecID id) { - return Singleton<TIds>()->Find(id); - } -} - + return Singleton<TIds>()->Find(id); + } +} + TCodedOutput::TCodedOutput(IOutputStream* out, const ICodec* c, size_t bufLen) - : C_(c) - , D_(bufLen) - , S_(out) -{ - if (bufLen > MAX_BUF_LEN) { + : C_(c) + , D_(bufLen) + , S_(out) +{ + if (bufLen > MAX_BUF_LEN) { ythrow yexception() << TStringBuf("too big buffer size: ") << bufLen; - } -} - + } +} + TCodedOutput::~TCodedOutput() { - try { - Finish(); - } catch (...) { - } -} - -void TCodedOutput::DoWrite(const void* buf, size_t len) { - const char* in = (const char*)buf; - - while (len) { - const size_t avail = D_.Avail(); - - if (len < avail) { - D_.Append(in, len); - - return; - } - - D_.Append(in, avail); - + try { + Finish(); + } catch (...) { + } +} + +void TCodedOutput::DoWrite(const void* buf, size_t len) { + const char* in = (const char*)buf; + + while (len) { + const size_t avail = D_.Avail(); + + if (len < avail) { + D_.Append(in, len); + + return; + } + + D_.Append(in, avail); + Y_ASSERT(!D_.Avail()); - - in += avail; - len -= avail; - + + in += avail; + len -= avail; + Y_VERIFY(FlushImpl(), "flush on writing failed"); - } -} - -bool TCodedOutput::FlushImpl() { - const bool ret = !D_.Empty(); - const size_t payload = sizeof(TCodecID) + sizeof(TBlockLen); - O_.Reserve(C_->MaxCompressedLength(D_) + payload); - - void* out = O_.Data() + payload; - const size_t olen = C_->Compress(D_, out); - - { - TMemoryOutput mo(O_.Data(), payload); - - ::Save(&mo, CodecID(C_)); - ::Save(&mo, SafeIntegerCast<TBlockLen>(olen)); - } - - S_->Write(O_.Data(), payload + olen); - - D_.Clear(); - O_.Clear(); - - return ret; -} - -void TCodedOutput::DoFlush() { - if (S_ && !D_.Empty()) { - FlushImpl(); - } -} - -void TCodedOutput::DoFinish() { - if (S_) { - Y_DEFER { + } +} + +bool TCodedOutput::FlushImpl() { + const bool ret = !D_.Empty(); + const size_t payload = sizeof(TCodecID) + sizeof(TBlockLen); + O_.Reserve(C_->MaxCompressedLength(D_) + payload); + + void* out = O_.Data() + payload; + const size_t olen = C_->Compress(D_, out); + + { + TMemoryOutput mo(O_.Data(), payload); + + ::Save(&mo, CodecID(C_)); + ::Save(&mo, SafeIntegerCast<TBlockLen>(olen)); + } + + S_->Write(O_.Data(), payload + olen); + + D_.Clear(); + O_.Clear(); + + return ret; +} + +void TCodedOutput::DoFlush() { + if (S_ && !D_.Empty()) { + FlushImpl(); + } +} + +void TCodedOutput::DoFinish() { + if (S_) { + Y_DEFER { S_ = nullptr; - }; - - if (FlushImpl()) { - //always write zero-length block as eos marker - FlushImpl(); - } - } -} - + }; + + if (FlushImpl()) { + //always write zero-length block as eos marker + FlushImpl(); + } + } +} + TDecodedInput::TDecodedInput(IInputStream* in) - : S_(in) - , C_(nullptr) -{ -} - -TDecodedInput::TDecodedInput(IInputStream* in, const ICodec* codec) - : S_(in) - , C_(codec) -{ -} - + : S_(in) + , C_(nullptr) +{ +} + +TDecodedInput::TDecodedInput(IInputStream* in, const ICodec* codec) + : S_(in) + , C_(codec) +{ +} + TDecodedInput::~TDecodedInput() = default; - + size_t TDecodedInput::DoUnboundedNext(const void** ptr) { - if (!S_) { + if (!S_) { return 0; - } - + } + TCodecID codecId; TBlockLen blockLen; - - { - const size_t payload = sizeof(TCodecID) + sizeof(TBlockLen); - char buf[32]; - + + { + const size_t payload = sizeof(TCodecID) + sizeof(TBlockLen); + char buf[32]; + S_->LoadOrFail(buf, payload); - - TMemoryInput in(buf, payload); - + + TMemoryInput in(buf, payload); + ::Load(&in, codecId); ::Load(&in, blockLen); - } - + } + if (!blockLen) { S_ = nullptr; - + return 0; - } - + } + if (Y_UNLIKELY(blockLen > 1024 * 1024 * 1024)) { - ythrow yexception() << "block size exceeds 1 GiB"; + ythrow yexception() << "block size exceeds 1 GiB"; } - TBuffer block; + TBuffer block; block.Resize(blockLen); - + S_->LoadOrFail(block.Data(), blockLen); - - auto codec = CodecByID(codecId); - - if (C_) { + + auto codec = CodecByID(codecId); + + if (C_) { Y_ENSURE(C_->Name() == codec->Name(), TStringBuf("incorrect stream codec")); - } - - if (codec->DecompressedLength(block) > MAX_BUF_LEN) { - ythrow yexception() << "broken stream"; - } - - codec->Decode(block, D_); - *ptr = D_.Data(); - + } + + if (codec->DecompressedLength(block) > MAX_BUF_LEN) { + ythrow yexception() << "broken stream"; + } + + codec->Decode(block, D_); + *ptr = D_.Data(); + return D_.Size(); -} +} diff --git a/library/cpp/blockcodecs/core/stream.h b/library/cpp/blockcodecs/core/stream.h index fd44ef88f2..b0d7929f05 100644 --- a/library/cpp/blockcodecs/core/stream.h +++ b/library/cpp/blockcodecs/core/stream.h @@ -1,46 +1,46 @@ -#pragma once - -#include <util/stream/walk.h> -#include <util/stream/input.h> -#include <util/stream/output.h> -#include <util/stream/zerocopy.h> -#include <util/generic/buffer.h> - -namespace NBlockCodecs { +#pragma once + +#include <util/stream/walk.h> +#include <util/stream/input.h> +#include <util/stream/output.h> +#include <util/stream/zerocopy.h> +#include <util/generic/buffer.h> + +namespace NBlockCodecs { struct ICodec; - + class TCodedOutput: public IOutputStream { - public: + public: TCodedOutput(IOutputStream* out, const ICodec* c, size_t bufLen); ~TCodedOutput() override; - - private: + + private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - - bool FlushImpl(); - - private: - const ICodec* C_; - TBuffer D_; - TBuffer O_; + + bool FlushImpl(); + + private: + const ICodec* C_; + TBuffer D_; + TBuffer O_; IOutputStream* S_; - }; - + }; + class TDecodedInput: public IWalkInput { - public: + public: TDecodedInput(IInputStream* in); - TDecodedInput(IInputStream* in, const ICodec* codec); - + TDecodedInput(IInputStream* in, const ICodec* codec); + ~TDecodedInput() override; - - private: + + private: size_t DoUnboundedNext(const void** ptr) override; - - private: - TBuffer D_; + + private: + TBuffer D_; IInputStream* S_; - const ICodec* C_; - }; -} + const ICodec* C_; + }; +} diff --git a/library/cpp/blockcodecs/core/ya.make b/library/cpp/blockcodecs/core/ya.make index 069e15927b..1956d7dc5e 100644 --- a/library/cpp/blockcodecs/core/ya.make +++ b/library/cpp/blockcodecs/core/ya.make @@ -1,10 +1,10 @@ -LIBRARY() - -OWNER(pg) - -SRCS( - codecs.cpp - stream.cpp -) - -END() +LIBRARY() + +OWNER(pg) + +SRCS( + codecs.cpp + stream.cpp +) + +END() diff --git a/library/cpp/blockcodecs/fuzz/main.cpp b/library/cpp/blockcodecs/fuzz/main.cpp index 763c6c5a10..388800face 100644 --- a/library/cpp/blockcodecs/fuzz/main.cpp +++ b/library/cpp/blockcodecs/fuzz/main.cpp @@ -4,17 +4,17 @@ #include <library/cpp/blockcodecs/codecs.h> #include <library/cpp/blockcodecs/fuzz/proto/case.pb.h> #include <library/cpp/blockcodecs/stream.h> - + #include <util/stream/input.h> #include <util/stream/length.h> -#include <util/stream/mem.h> +#include <util/stream/mem.h> #include <util/stream/null.h> #include <util/stream/str.h> - + using NBlockCodecs::NFuzz::TPackUnpackCase; using NBlockCodecs::TCodedOutput; using NBlockCodecs::TDecodedInput; - + static void ValidateBufferSize(const ui32 size) { Y_ENSURE(size > 0 && size <= 16ULL * 1024); } @@ -22,14 +22,14 @@ static void ValidateBufferSize(const ui32 size) { static void DoOnlyDecode(const TPackUnpackCase& case_) { if (!case_.GetPacked()) { return; - } - + } + TMemoryInput mi(case_.GetData().data(), case_.GetData().size()); TDecodedInput di(&mi); TNullOutput no; TCountingOutput cno(&no); TransferData(&di, &cno); -} +} static void DoDecodeEncode(const TPackUnpackCase& case_) { auto* const codec = NBlockCodecs::Codec(case_.GetCodecName()); diff --git a/library/cpp/blockcodecs/fuzz/ya.make b/library/cpp/blockcodecs/fuzz/ya.make index bc8becc9e1..a8f6d9f617 100644 --- a/library/cpp/blockcodecs/fuzz/ya.make +++ b/library/cpp/blockcodecs/fuzz/ya.make @@ -2,12 +2,12 @@ OWNER( pg g:util ) - + IF (NOT MSVC) FUZZ() - + SIZE(MEDIUM) - + SRCS( main.cpp ) diff --git a/library/cpp/blockcodecs/ut/ya.make b/library/cpp/blockcodecs/ut/ya.make index 25b882c15b..b6de366c43 100644 --- a/library/cpp/blockcodecs/ut/ya.make +++ b/library/cpp/blockcodecs/ut/ya.make @@ -2,14 +2,14 @@ UNITTEST_FOR(library/cpp/blockcodecs) OWNER(pg) -FORK_TESTS() +FORK_TESTS() -FORK_SUBTESTS() +FORK_SUBTESTS() -SPLIT_FACTOR(40) - -TIMEOUT(300) +SPLIT_FACTOR(40) +TIMEOUT(300) + SIZE(MEDIUM) SRCS( diff --git a/library/cpp/bucket_quoter/bucket_quoter.cpp b/library/cpp/bucket_quoter/bucket_quoter.cpp index e7f8b1a869..14f4565644 100644 --- a/library/cpp/bucket_quoter/bucket_quoter.cpp +++ b/library/cpp/bucket_quoter/bucket_quoter.cpp @@ -1 +1 @@ -#include "bucket_quoter.h" +#include "bucket_quoter.h" diff --git a/library/cpp/bucket_quoter/bucket_quoter.h b/library/cpp/bucket_quoter/bucket_quoter.h index 3d92ef8450..494644c26a 100644 --- a/library/cpp/bucket_quoter/bucket_quoter.h +++ b/library/cpp/bucket_quoter/bucket_quoter.h @@ -52,7 +52,7 @@ struct TInstantTimerMs { struct THPTimerUs { using TTime = NHPTimer::STime; - static constexpr ui64 Resolution = 1000000ull; // microseconds + static constexpr ui64 Resolution = 1000000ull; // microseconds static TTime Now() { NHPTimer::STime ret; NHPTimer::GetTime(&ret); @@ -88,7 +88,7 @@ public: , TokensUsed(tokensUsed) , UsecWaited(usecWaited) , AggregateInflow(aggregateInflow) - , Bucket(fill ? capacity : 0) + , Bucket(fill ? capacity : 0) , LastAdd(Timer::Now()) , InflowTokensPerSecond(&FixedInflow) , BucketTokensCapacity(&FixedCapacity) @@ -107,7 +107,7 @@ public: , TokensUsed(tokensUsed) , UsecWaited(usecWaited) , AggregateInflow(aggregateInflow) - , Bucket(fill ? AtomicGet(*capacity) : 0) + , Bucket(fill ? AtomicGet(*capacity) : 0) , LastAdd(Timer::Now()) , InflowTokensPerSecond(inflow) , BucketTokensCapacity(capacity) @@ -214,7 +214,7 @@ public: } void Sleep() { - while (!IsAvail()) { + while (!IsAvail()) { ui32 delay = GetWaitTime(); if (delay != 0) { usleep(delay); @@ -263,9 +263,9 @@ private: } } - StatCounter* MsgPassed; - StatCounter* BucketUnderflows; - StatCounter* TokensUsed; + StatCounter* MsgPassed; + StatCounter* BucketUnderflows; + StatCounter* TokensUsed; StatCounter* UsecWaited; StatCounter* AggregateInflow; @@ -274,8 +274,8 @@ private: Lock BucketMutex; ui64 Seqno = 0; - TAtomic* InflowTokensPerSecond; - TAtomic* BucketTokensCapacity; + TAtomic* InflowTokensPerSecond; + TAtomic* BucketTokensCapacity; TAtomic FixedInflow; TAtomic FixedCapacity; }; diff --git a/library/cpp/build_info/build_info.h b/library/cpp/build_info/build_info.h index a494870ba3..e23c85e8cb 100644 --- a/library/cpp/build_info/build_info.h +++ b/library/cpp/build_info/build_info.h @@ -7,7 +7,7 @@ extern "C" { #endif -const char* GetBuildType(); +const char* GetBuildType(); #if defined(__cplusplus) } diff --git a/library/cpp/build_info/build_info_static.h b/library/cpp/build_info/build_info_static.h index 9bf9939dd9..75d3916e69 100644 --- a/library/cpp/build_info/build_info_static.h +++ b/library/cpp/build_info/build_info_static.h @@ -4,9 +4,9 @@ extern "C" { #endif -const char* GetCompilerVersion(); -const char* GetCompilerFlags(); // "-std=c++14 -DNDEBUG -O2 -m64 ..." -const char* GetBuildInfo(); // Compiler version and flags +const char* GetCompilerVersion(); +const char* GetCompilerFlags(); // "-std=c++14 -DNDEBUG -O2 -m64 ..." +const char* GetBuildInfo(); // Compiler version and flags #if defined(__cplusplus) } diff --git a/library/cpp/build_info/sandbox.h b/library/cpp/build_info/sandbox.h index 924ff89c8c..a1199674d6 100644 --- a/library/cpp/build_info/sandbox.h +++ b/library/cpp/build_info/sandbox.h @@ -4,7 +4,7 @@ extern "C" { #endif -const char* GetSandboxTaskId(); +const char* GetSandboxTaskId(); const char* GetKosherSvnVersion(); #if defined(__cplusplus) diff --git a/library/cpp/build_info/ya.make b/library/cpp/build_info/ya.make index 99886a8893..b587497c3b 100644 --- a/library/cpp/build_info/ya.make +++ b/library/cpp/build_info/ya.make @@ -1,4 +1,4 @@ -LIBRARY() +LIBRARY() OWNER( mvel diff --git a/library/cpp/cache/cache.h b/library/cpp/cache/cache.h index 6dc997076d..9662601209 100644 --- a/library/cpp/cache/cache.h +++ b/library/cpp/cache/cache.h @@ -1,7 +1,7 @@ #pragma once #include <util/generic/algorithm.h> -#include <util/generic/ptr.h> +#include <util/generic/ptr.h> #include <util/generic/intrlist.h> #include <util/generic/hash_set.h> #include <util/generic/vector.h> @@ -28,7 +28,7 @@ public: } public: - struct TItem: public TIntrusiveListItem<TItem> { + struct TItem: public TIntrusiveListItem<TItem> { typedef TIntrusiveListItem<TItem> TBase; TItem(const TKey& key, const TValue& value = TValue()) : TBase() @@ -44,11 +44,11 @@ public: { } - bool operator<(const TItem& rhs) const { + bool operator<(const TItem& rhs) const { return Key < rhs.Key; } - bool operator==(const TItem& rhs) const { + bool operator==(const TItem& rhs) const { return Key == rhs.Key; } @@ -56,7 +56,7 @@ public: TValue Value; struct THash { - size_t operator()(const TItem& item) const { + size_t operator()(const TItem& item) const { return ::THash<TKey>()(item.Key); } }; @@ -100,7 +100,7 @@ public: size_t GetSize() const { return ItemsAmount; } - + size_t GetTotalSize() const { return TotalSize; } @@ -134,7 +134,7 @@ public: { } - struct TItem: public TIntrusiveListItem<TItem> { + struct TItem: public TIntrusiveListItem<TItem> { typedef TIntrusiveListItem<TItem> TBase; TItem(const TKey& key, const TValue& value = TValue()) : TBase() @@ -152,11 +152,11 @@ public: { } - bool operator<(const TItem& rhs) const { + bool operator<(const TItem& rhs) const { return Key < rhs.Key; } - bool operator==(const TItem& rhs) const { + bool operator==(const TItem& rhs) const { return Key == rhs.Key; } @@ -165,7 +165,7 @@ public: size_t Counter; struct THash { - size_t operator()(const TItem& item) const { + size_t operator()(const TItem& item) const { return ::THash<TKey>()(item.Key); } }; @@ -256,11 +256,11 @@ public: { } - bool operator<(const TItem& rhs) const { + bool operator<(const TItem& rhs) const { return Key < rhs.Key; } - bool operator==(const TItem& rhs) const { + bool operator==(const TItem& rhs) const { return Key == rhs.Key; } @@ -269,7 +269,7 @@ public: TWeight Weight; struct THash { - size_t operator()(const TItem& item) const { + size_t operator()(const TItem& item) const { return ::THash<TKey>()(item.Key); } }; @@ -380,7 +380,7 @@ class TCache { typedef THashMultiSet<TItem, THash> TIndex; typedef typename TIndex::iterator TIndexIterator; typedef typename TIndex::const_iterator TIndexConstIterator; - + public: class TIterator { public: @@ -389,23 +389,23 @@ public: { } - TValue& operator*() { + TValue& operator*() { return const_cast<TValue&>(Iter->Value); } - TValue* operator->() { + TValue* operator->() { return const_cast<TValue*>(&Iter->Value); } - bool operator==(const TIterator& rhs) const { + bool operator==(const TIterator& rhs) const { return Iter == rhs.Iter; } - bool operator!=(const TIterator& rhs) const { + bool operator!=(const TIterator& rhs) const { return Iter != rhs.Iter; } - TIterator& operator++() { + TIterator& operator++() { ++Iter; return *this; } @@ -419,7 +419,7 @@ public: } friend class TCache<TKey, TValue, TListType, TDeleter>; - + private: TIndexConstIterator Iter; }; @@ -459,7 +459,7 @@ public: } // note: it shouldn't touch 'value' if it returns false. - bool PickOut(const TKey& key, TValue* value) { + bool PickOut(const TKey& key, TValue* value) { Y_ASSERT(value); TIndexIterator it = Index.find(TItem(key)); if (it == Index.end()) @@ -475,7 +475,7 @@ public: return Insert(p.first, p.second); } - bool Insert(const TKey& key, const TValue& value) { + bool Insert(const TKey& key, const TValue& value) { TItem tmpItem(key, value); if (!MultiValue && Index.find(tmpItem) != Index.end()) return false; @@ -550,7 +550,7 @@ protected: TListType List; bool MultiValue; - TIterator FindByItem(TItem* item) { + TIterator FindByItem(TItem* item) { std::pair<TIndexIterator, TIndexIterator> p = Index.equal_range(*item); // we have to delete the exact unlinked item (there may be multiple items for one key) TIndexIterator it; @@ -593,7 +593,7 @@ public: } TIterator FindOldest() { - return TBase::Empty() ? TBase::End() : this->FindByItem(TBase::List.GetOldest()); + return TBase::Empty() ? TBase::End() : this->FindByItem(TBase::List.GetOldest()); } size_t TotalSize() const { @@ -602,8 +602,8 @@ public: }; template <typename TKey, typename TValue, typename TDeleter = TNoopDelete> -class TLFUCache: public TCache<TKey, TValue, TLFUList<TKey, TValue>, TDeleter> { - typedef TCache<TKey, TValue, TLFUList<TKey, TValue>, TDeleter> TBase; +class TLFUCache: public TCache<TKey, TValue, TLFUList<TKey, TValue>, TDeleter> { + typedef TCache<TKey, TValue, TLFUList<TKey, TValue>, TDeleter> TBase; using TListType = TLFUList<TKey, TValue>; public: @@ -627,7 +627,7 @@ public: // discards the least weighted items first // doesn't support promotion template <typename TKey, typename TValue, typename TWeight, typename TWeighter, typename TDeleter = TNoopDelete> -class TLWCache: public TCache<TKey, TValue, TLWList<TKey, TValue, TWeight, TWeighter>, TDeleter> { +class TLWCache: public TCache<TKey, TValue, TLWList<TKey, TValue, TWeight, TWeighter>, TDeleter> { typedef TCache<TKey, TValue, TLWList<TKey, TValue, TWeight, TWeighter>, TDeleter> TBase; using TListType = TLWList<TKey, TValue, TWeight, TWeighter>; diff --git a/library/cpp/cache/thread_safe_cache.h b/library/cpp/cache/thread_safe_cache.h index 71e1442717..276ac28bc6 100644 --- a/library/cpp/cache/thread_safe_cache.h +++ b/library/cpp/cache/thread_safe_cache.h @@ -33,8 +33,8 @@ namespace NPrivate { TThreadSafeCache(const ICallbacks& callbacks, size_t maxSize = Max<size_t>()) : Callbacks(callbacks) , Cache(maxSize) - { - } + { + } bool Insert(const Key& key, const TPtr& value) { if (!Contains(key)) { @@ -81,12 +81,12 @@ namespace NPrivate { return iter != Cache.End(); } - template <class TCallbacks> + template <class TCallbacks> static const TPtr Get(TArgs... args) { return TThreadSafeCacheSingleton<TCallbacks>::Get(args...); } - template <class TCallbacks> + template <class TCallbacks> static const TPtr Erase(TArgs... args) { return TThreadSafeCacheSingleton<TCallbacks>::Erase(args...); } @@ -154,8 +154,8 @@ namespace NPrivate { TThreadSafeCacheSingleton() : Cache(Callbacks) - { - } + { + } private: TCallbacks Callbacks; @@ -177,7 +177,7 @@ namespace NPrivate { }; template <class TKey, class TValue> - using TListType = TLWList<TKey, TValue, int, TConstWeighter<TValue>>; + using TListType = TLWList<TKey, TValue, int, TConstWeighter<TValue>>; template <class TKey, class TValue, class... TArgs> using TCache = TThreadSafeCache<TKey, TValue, TListType, EGettersPromotionPolicy::Unpromoted, TArgs...>; diff --git a/library/cpp/cache/ut/cache_ut.cpp b/library/cpp/cache/ut/cache_ut.cpp index 329872cfde..d7fabfffee 100644 --- a/library/cpp/cache/ut/cache_ut.cpp +++ b/library/cpp/cache/ut/cache_ut.cpp @@ -1,7 +1,7 @@ #include <library/cpp/cache/cache.h> #include <library/cpp/cache/thread_safe_cache.h> #include <library/cpp/testing/unittest/registar.h> - + struct TStrokaWeighter { static size_t Weight(const TString& s) { return s.size(); @@ -10,23 +10,23 @@ struct TStrokaWeighter { Y_UNIT_TEST_SUITE(TCacheTest) { Y_UNIT_TEST(LRUListTest) { - typedef TLRUList<int, TString> TListType; - TListType list(2); + typedef TLRUList<int, TString> TListType; + TListType list(2); - TListType::TItem x1(1, "ttt"); - list.Insert(&x1); - UNIT_ASSERT_EQUAL(list.GetOldest()->Key, 1); + TListType::TItem x1(1, "ttt"); + list.Insert(&x1); + UNIT_ASSERT_EQUAL(list.GetOldest()->Key, 1); - TListType::TItem x2(2, "yyy"); - list.Insert(&x2); - UNIT_ASSERT_EQUAL(list.GetOldest()->Key, 1); + TListType::TItem x2(2, "yyy"); + list.Insert(&x2); + UNIT_ASSERT_EQUAL(list.GetOldest()->Key, 1); - list.Promote(list.GetOldest()); - UNIT_ASSERT_EQUAL(list.GetOldest()->Key, 2); + list.Promote(list.GetOldest()); + UNIT_ASSERT_EQUAL(list.GetOldest()->Key, 2); - TListType::TItem x3(3, "zzz"); - list.Insert(&x3); - UNIT_ASSERT_EQUAL(list.GetOldest()->Key, 1); + TListType::TItem x3(3, "zzz"); + list.Insert(&x3); + UNIT_ASSERT_EQUAL(list.GetOldest()->Key, 1); } Y_UNIT_TEST(LRUListWeightedTest) { @@ -66,52 +66,52 @@ Y_UNIT_TEST_SUITE(TCacheTest) { } Y_UNIT_TEST(LFUListTest) { - typedef TLFUList<int, TString> TListType; - TListType list(2); + typedef TLFUList<int, TString> TListType; + TListType list(2); - TListType::TItem x1(1, "ttt"); - list.Insert(&x1); - UNIT_ASSERT_EQUAL(list.GetLeastFrequentlyUsed()->Key, 1); + TListType::TItem x1(1, "ttt"); + list.Insert(&x1); + UNIT_ASSERT_EQUAL(list.GetLeastFrequentlyUsed()->Key, 1); - TListType::TItem x2(2, "yyy"); - list.Insert(&x2); - UNIT_ASSERT_EQUAL(list.GetLeastFrequentlyUsed()->Key, 1); + TListType::TItem x2(2, "yyy"); + list.Insert(&x2); + UNIT_ASSERT_EQUAL(list.GetLeastFrequentlyUsed()->Key, 1); - list.Promote(list.GetLeastFrequentlyUsed()); - UNIT_ASSERT_EQUAL(list.GetLeastFrequentlyUsed()->Key, 2); + list.Promote(list.GetLeastFrequentlyUsed()); + UNIT_ASSERT_EQUAL(list.GetLeastFrequentlyUsed()->Key, 2); - TListType::TItem x3(3, "zzz"); - list.Insert(&x3); - UNIT_ASSERT_EQUAL(list.GetLeastFrequentlyUsed()->Key, 1); + TListType::TItem x3(3, "zzz"); + list.Insert(&x3); + UNIT_ASSERT_EQUAL(list.GetLeastFrequentlyUsed()->Key, 1); } Y_UNIT_TEST(LWListTest) { - typedef TLWList<int, TString, size_t, TStrokaWeighter> TListType; - TListType list(2); - - TListType::TItem x1(1, "tt"); - list.Insert(&x1); - UNIT_ASSERT_EQUAL(list.GetLightest()->Key, 1); - UNIT_ASSERT_EQUAL(list.GetSize(), 1); - - TListType::TItem x2(2, "yyyy"); - list.Insert(&x2); - UNIT_ASSERT_EQUAL(list.GetLightest()->Key, 1); - UNIT_ASSERT_EQUAL(list.GetSize(), 2); - - TListType::TItem x3(3, "z"); - list.Insert(&x3); - UNIT_ASSERT_EQUAL(list.GetLightest()->Key, 1); - UNIT_ASSERT_EQUAL(list.GetSize(), 2); - - TListType::TItem x4(4, "xxxxxx"); - list.Insert(&x4); - UNIT_ASSERT_EQUAL(list.GetLightest()->Key, 2); - UNIT_ASSERT_EQUAL(list.GetSize(), 2); - - list.Erase(&x2); - UNIT_ASSERT_EQUAL(list.GetLightest()->Key, 4); - UNIT_ASSERT_EQUAL(list.GetSize(), 1); + typedef TLWList<int, TString, size_t, TStrokaWeighter> TListType; + TListType list(2); + + TListType::TItem x1(1, "tt"); + list.Insert(&x1); + UNIT_ASSERT_EQUAL(list.GetLightest()->Key, 1); + UNIT_ASSERT_EQUAL(list.GetSize(), 1); + + TListType::TItem x2(2, "yyyy"); + list.Insert(&x2); + UNIT_ASSERT_EQUAL(list.GetLightest()->Key, 1); + UNIT_ASSERT_EQUAL(list.GetSize(), 2); + + TListType::TItem x3(3, "z"); + list.Insert(&x3); + UNIT_ASSERT_EQUAL(list.GetLightest()->Key, 1); + UNIT_ASSERT_EQUAL(list.GetSize(), 2); + + TListType::TItem x4(4, "xxxxxx"); + list.Insert(&x4); + UNIT_ASSERT_EQUAL(list.GetLightest()->Key, 2); + UNIT_ASSERT_EQUAL(list.GetSize(), 2); + + list.Erase(&x2); + UNIT_ASSERT_EQUAL(list.GetLightest()->Key, 4); + UNIT_ASSERT_EQUAL(list.GetSize(), 1); } Y_UNIT_TEST(SimpleTest) { @@ -139,7 +139,7 @@ Y_UNIT_TEST_SUITE(TCacheTest) { TCache::TIterator it = s.Find(3); s.Erase(it); UNIT_ASSERT(s.Find(3) == s.End()); - } + } Y_UNIT_TEST(LRUWithCustomSizeProviderTest) { typedef TLRUCache<int, TString, TNoopDelete, size_t(*)(const TString&)> TCache; @@ -322,39 +322,39 @@ Y_UNIT_TEST_SUITE(TCacheTest) { UNIT_ASSERT(*s.Find(1) == "bcde"); // (1, "bcde") will be promoted UNIT_ASSERT(*s.FindOldest() == "fghi"); - } + } - struct TMyDelete { - static int count; - template <typename T> - static void Destroy(const T&) { - ++count; - } - }; - int TMyDelete::count = 0; + struct TMyDelete { + static int count; + template <typename T> + static void Destroy(const T&) { + ++count; + } + }; + int TMyDelete::count = 0; Y_UNIT_TEST(DeleterTest) { - typedef TLRUCache<int, TString, TMyDelete> TCache; - TCache s(2); - s.Insert(1, "123"); - s.Insert(2, "456"); - s.Insert(3, "789"); - UNIT_ASSERT(TMyDelete::count == 1); - TCache::TIterator it = s.Find(2); - UNIT_ASSERT(it != s.End()); - s.Erase(it); - UNIT_ASSERT(TMyDelete::count == 2); - } + typedef TLRUCache<int, TString, TMyDelete> TCache; + TCache s(2); + s.Insert(1, "123"); + s.Insert(2, "456"); + s.Insert(3, "789"); + UNIT_ASSERT(TMyDelete::count == 1); + TCache::TIterator it = s.Find(2); + UNIT_ASSERT(it != s.End()); + s.Erase(it); + UNIT_ASSERT(TMyDelete::count == 2); + } Y_UNIT_TEST(PromoteOnFind) { - typedef TLRUCache<int, TString> TCache; - TCache s(2); - s.Insert(1, "123"); - s.Insert(2, "456"); - UNIT_ASSERT(s.Find(1) != s.End()); - s.Insert(3, "789"); - UNIT_ASSERT(s.Find(1) != s.End()); // Key 2 should have been deleted - } + typedef TLRUCache<int, TString> TCache; + TCache s(2); + s.Insert(1, "123"); + s.Insert(2, "456"); + UNIT_ASSERT(s.Find(1) != s.End()); + s.Insert(3, "789"); + UNIT_ASSERT(s.Find(1) != s.End()); // Key 2 should have been deleted + } } Y_UNIT_TEST_SUITE(TThreadSafeCacheTest) { @@ -362,7 +362,7 @@ Y_UNIT_TEST_SUITE(TThreadSafeCacheTest) { const char* VALS[] = {"abcd", "defg", "hjkl"}; - class TCallbacks: public TCache::ICallbacks { + class TCallbacks: public TCache::ICallbacks { public: TKey GetKey(ui32 i) const override { return i; @@ -380,7 +380,7 @@ Y_UNIT_TEST_SUITE(TThreadSafeCacheTest) { const TString data = *TCache::Get<TCallbacks>(i); UNIT_ASSERT(data == VALS[i]); } - } + } Y_UNIT_TEST(InsertUpdateTest) { TCallbacks callbacks; diff --git a/library/cpp/cgiparam/cgiparam.cpp b/library/cpp/cgiparam/cgiparam.cpp index f3277b8e4b..89efbe4c1d 100644 --- a/library/cpp/cgiparam/cgiparam.cpp +++ b/library/cpp/cgiparam/cgiparam.cpp @@ -1,8 +1,8 @@ -#include "cgiparam.h" - +#include "cgiparam.h" + #include <library/cpp/string_utils/scan/scan.h> #include <library/cpp/string_utils/quote/quote.h> - + #include <util/generic/singleton.h> TCgiParameters::TCgiParameters(std::initializer_list<std::pair<TString, TString>> il) { @@ -12,21 +12,21 @@ TCgiParameters::TCgiParameters(std::initializer_list<std::pair<TString, TString> } const TString& TCgiParameters::Get(const TStringBuf name, size_t numOfValue) const noexcept { - const auto it = Find(name, numOfValue); - + const auto it = Find(name, numOfValue); + return end() == it ? Default<TString>() : it->second; -} - +} + bool TCgiParameters::Erase(const TStringBuf name, size_t pos) { - const auto pair = equal_range(name); - - for (auto it = pair.first; it != pair.second; ++it, --pos) { + const auto pair = equal_range(name); + + for (auto it = pair.first; it != pair.second; ++it, --pos) { if (0 == pos) { erase(it); return true; } - } - + } + return false; } @@ -48,31 +48,31 @@ bool TCgiParameters::Erase(const TStringBuf name, const TStringBuf val) { size_t TCgiParameters::EraseAll(const TStringBuf name) { size_t num = 0; - - const auto pair = equal_range(name); - - for (auto it = pair.first; it != pair.second; erase(it++), ++num) + + const auto pair = equal_range(name); + + for (auto it = pair.first; it != pair.second; erase(it++), ++num) ; - + return num; } void TCgiParameters::JoinUnescaped(const TStringBuf key, char sep, TStringBuf val) { - const auto pair = equal_range(key); - auto it = pair.first; - + const auto pair = equal_range(key); + auto it = pair.first; + if (it == pair.second) { // not found - if (val.IsInited()) { + if (val.IsInited()) { emplace_hint(it, TString(key), TString(val)); - } - } else { + } + } else { TString& dst = it->second; - - for (++it; it != pair.second; erase(it++)) { + + for (++it; it != pair.second; erase(it++)) { dst += sep; dst.AppendNoAlias(it->second.data(), it->second.size()); } - + if (val.IsInited()) { dst += sep; dst += val; @@ -85,44 +85,44 @@ static inline TString DoUnescape(const TStringBuf s) { res.reserve(CgiUnescapeBufLen(s.size())); res.ReserveAndResize(CgiUnescape(res.begin(), s).size()); - - return res; -} + + return res; +} void TCgiParameters::InsertEscaped(const TStringBuf name, const TStringBuf value) { InsertUnescaped(DoUnescape(name), DoUnescape(value)); } -template <bool addAll, class F> +template <bool addAll, class F> static inline void DoScan(const TStringBuf s, F& f) { - ScanKeyValue<addAll, '&', '='>(s, f); -} - -struct TAddEscaped { - TCgiParameters* C; - + ScanKeyValue<addAll, '&', '='>(s, f); +} + +struct TAddEscaped { + TCgiParameters* C; + inline void operator()(const TStringBuf key, const TStringBuf val) { C->InsertEscaped(key, val); - } -}; - + } +}; + void TCgiParameters::Scan(const TStringBuf query, bool form) { - Flush(); + Flush(); form ? ScanAdd(query) : ScanAddAll(query); } void TCgiParameters::ScanAdd(const TStringBuf query) { TAddEscaped f = {this}; - - DoScan<false>(query, f); + + DoScan<false>(query, f); } void TCgiParameters::ScanAddUnescaped(const TStringBuf query) { auto f = [this](const TStringBuf key, const TStringBuf val) { this->InsertUnescaped(key, val); - }; + }; - DoScan<false>(query, f); + DoScan<false>(query, f); } void TCgiParameters::ScanAddAllUnescaped(const TStringBuf query) { @@ -135,48 +135,48 @@ void TCgiParameters::ScanAddAllUnescaped(const TStringBuf query) { void TCgiParameters::ScanAddAll(const TStringBuf query) { TAddEscaped f = {this}; - - DoScan<true>(query, f); + + DoScan<true>(query, f); } TString TCgiParameters::Print() const { TString res; - - res.reserve(PrintSize()); - const char* end = Print(res.begin()); + + res.reserve(PrintSize()); + const char* end = Print(res.begin()); res.ReserveAndResize(end - res.data()); - - return res; + + return res; } -char* TCgiParameters::Print(char* res) const { - if (empty()) { - return res; - } - - for (auto i = begin();;) { +char* TCgiParameters::Print(char* res) const { + if (empty()) { + return res; + } + + for (auto i = begin();;) { res = CGIEscape(res, i->first); - *res++ = '='; + *res++ = '='; res = CGIEscape(res, i->second); - - if (++i == end()) { + + if (++i == end()) { break; - } - - *res++ = '&'; - } - + } + + *res++ = '&'; + } + return res; } size_t TCgiParameters::PrintSize() const noexcept { - size_t res = size(); // for '&' - + size_t res = size(); // for '&' + for (const auto& i : *this) { res += CgiEscapeBufLen(i.first.size() + i.second.size()); // extra zero will be used for '=' - } - - return res; + } + + return res; } TString TCgiParameters::QuotedPrint(const char* safe) const { @@ -205,27 +205,27 @@ TString TCgiParameters::QuotedPrint(const char* safe) const { } TCgiParameters::const_iterator TCgiParameters::Find(const TStringBuf name, size_t pos) const noexcept { - const auto pair = equal_range(name); - - for (auto it = pair.first; it != pair.second; ++it, --pos) { - if (0 == pos) { + const auto pair = equal_range(name); + + for (auto it = pair.first; it != pair.second; ++it, --pos) { + if (0 == pos) { return it; - } - } - + } + } + return end(); } bool TCgiParameters::Has(const TStringBuf name, const TStringBuf value) const noexcept { - const auto pair = equal_range(name); - - for (auto it = pair.first; it != pair.second; ++it) { - if (value == it->second) { - return true; - } - } - - return false; + const auto pair = equal_range(name); + + for (auto it = pair.first; it != pair.second; ++it) { + if (value == it->second) { + return true; + } + } + + return false; } TQuickCgiParam::TQuickCgiParam(const TStringBuf cgiParamStr) { diff --git a/library/cpp/cgiparam/cgiparam.h b/library/cpp/cgiparam/cgiparam.h index 87d1ab0ad4..f4ee174213 100644 --- a/library/cpp/cgiparam/cgiparam.h +++ b/library/cpp/cgiparam/cgiparam.h @@ -6,16 +6,16 @@ #include <util/generic/map.h> #include <util/generic/strbuf.h> #include <util/generic/string.h> - + #include <initializer_list> - -struct TStringLess { - template <class T1, class T2> + +struct TStringLess { + template <class T1, class T2> inline bool operator()(const T1& t1, const T2& t2) const noexcept { - return TStringBuf(t1) < TStringBuf(t2); - } -}; - + return TStringBuf(t1) < TStringBuf(t2); + } +}; + class TCgiParameters: public TMultiMap<TString, TString> { public: TCgiParameters() = default; @@ -45,7 +45,7 @@ public: void ScanAddUnescaped(const TStringBuf cgiParStr); void ScanAddAllUnescaped(const TStringBuf cgiParStr); void ScanAddAll(const TStringBuf cgiParStr); - + /// Returns the string representation of all the stored parameters /** * @note The returned string has format <name1>=<value1>&<name2>=<value2>&... @@ -84,9 +84,9 @@ public: */ Y_PURE_FUNCTION const TString& Get(const TStringBuf name, size_t numOfValue = 0) const noexcept; - + void InsertEscaped(const TStringBuf name, const TStringBuf value); - + #if !defined(__GLIBCXX__) template <typename TName, typename TValue> inline void InsertUnescaped(TName&& name, TValue&& value) { @@ -102,7 +102,7 @@ public: emplace(TString(name), TString(value)); } #endif - + // replace all values for a given key with new values template <typename TIter> void ReplaceUnescaped(const TStringBuf key, TIter valuesBegin, const TIter valuesEnd); @@ -112,7 +112,7 @@ public: } void ReplaceUnescaped(const TStringBuf key, const TStringBuf value) { - ReplaceUnescaped(key, {value}); + ReplaceUnescaped(key, {value}); } // join multiple values into a single one using a separator @@ -121,13 +121,13 @@ public: bool Erase(const TStringBuf name, size_t numOfValue = 0); bool Erase(const TStringBuf name, const TStringBuf val); - + inline const char* FormField(const TStringBuf name, size_t numOfValue = 0) const { const_iterator it = Find(name, numOfValue); - + if (it == end()) { return nullptr; - } + } return it->second.data(); } @@ -139,18 +139,18 @@ void TCgiParameters::ReplaceUnescaped(const TStringBuf key, TIter valuesBegin, c auto current = oldRange.first; // reuse as many existing nodes as possible (probably none) - for (; valuesBegin != valuesEnd && current != oldRange.second; ++valuesBegin, ++current) { + for (; valuesBegin != valuesEnd && current != oldRange.second; ++valuesBegin, ++current) { current->second = *valuesBegin; } // if there were more nodes than we need to insert then erase remaining ones - for (; current != oldRange.second; erase(current++)) { + for (; current != oldRange.second; erase(current++)) { } // if there were less nodes than we need to insert then emplace the rest of the range if (valuesBegin != valuesEnd) { const TString keyStr = TString(key); - for (; valuesBegin != valuesEnd; ++valuesBegin) { + for (; valuesBegin != valuesEnd; ++valuesBegin) { emplace_hint(oldRange.second, keyStr, TString(*valuesBegin)); } } diff --git a/library/cpp/cgiparam/cgiparam_ut.cpp b/library/cpp/cgiparam/cgiparam_ut.cpp index a562342084..f7e6ddf6be 100644 --- a/library/cpp/cgiparam/cgiparam_ut.cpp +++ b/library/cpp/cgiparam/cgiparam_ut.cpp @@ -1,22 +1,22 @@ -#include "cgiparam.h" - +#include "cgiparam.h" + #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TCgiParametersTest) { Y_UNIT_TEST(TestScan1) { - TCgiParameters C; - C.Scan("aaa=b%62b&ccc=ddd&ag0="); - UNIT_ASSERT_EQUAL(C.Get("aaa") == "bbb", true); - UNIT_ASSERT_EQUAL(C.NumOfValues("ag0") == 1, true); - UNIT_ASSERT(C.Has("ccc", "ddd")); - UNIT_ASSERT(C.Has("ag0", "")); - UNIT_ASSERT(!C.Has("a", "bbb")); - UNIT_ASSERT(!C.Has("aaa", "bb")); + TCgiParameters C; + C.Scan("aaa=b%62b&ccc=ddd&ag0="); + UNIT_ASSERT_EQUAL(C.Get("aaa") == "bbb", true); + UNIT_ASSERT_EQUAL(C.NumOfValues("ag0") == 1, true); + UNIT_ASSERT(C.Has("ccc", "ddd")); + UNIT_ASSERT(C.Has("ag0", "")); + UNIT_ASSERT(!C.Has("a", "bbb")); + UNIT_ASSERT(!C.Has("aaa", "bb")); UNIT_ASSERT(C.Has("ccc")); UNIT_ASSERT(!C.Has("zzzzzz")); - } - + } + Y_UNIT_TEST(TestQuick) { TQuickCgiParam C("aaa=b%62b&ccc=ddd&ag0="); UNIT_ASSERT_EQUAL(C.Get("aaa") == "bbb", true); @@ -40,47 +40,47 @@ Y_UNIT_TEST_SUITE(TCgiParametersTest) { Y_UNIT_TEST(TestScan2) { const TString parsee("=000&aaa=bbb&ag0=&ccc=ddd"); - TCgiParameters c; - c.Scan(parsee); - - UNIT_ASSERT_VALUES_EQUAL(c.Print(), parsee); - } - + TCgiParameters c; + c.Scan(parsee); + + UNIT_ASSERT_VALUES_EQUAL(c.Print(), parsee); + } + Y_UNIT_TEST(TestScan3) { const TString parsee("aaa=bbb&ag0=&ccc=ddd"); - TCgiParameters c; - c.Scan(parsee); - + TCgiParameters c; + c.Scan(parsee); + c.InsertUnescaped("d", "xxx"); - - UNIT_ASSERT_VALUES_EQUAL(c.Print(), parsee + "&d=xxx"); - } - + + UNIT_ASSERT_VALUES_EQUAL(c.Print(), parsee + "&d=xxx"); + } + Y_UNIT_TEST(TestScanAddAll1) { - TCgiParameters c; - c.ScanAddAll("qw"); - + TCgiParameters c; + c.ScanAddAll("qw"); + UNIT_ASSERT_VALUES_EQUAL(c.size(), 1u); - UNIT_ASSERT(c.Get("qw").empty()); - } - + UNIT_ASSERT(c.Get("qw").empty()); + } + Y_UNIT_TEST(TestScanAddAll2) { - TCgiParameters c; - c.ScanAddAll("qw&"); - + TCgiParameters c; + c.ScanAddAll("qw&"); + UNIT_ASSERT_VALUES_EQUAL(c.size(), 1u); - UNIT_ASSERT(c.Get("qw").empty()); - } - + UNIT_ASSERT(c.Get("qw").empty()); + } + Y_UNIT_TEST(TestScanAddAll3) { - TCgiParameters c; - c.ScanAddAll("qw=1&x"); - + TCgiParameters c; + c.ScanAddAll("qw=1&x"); + UNIT_ASSERT_VALUES_EQUAL(c.size(), 2u); - UNIT_ASSERT_VALUES_EQUAL(c.Get("qw"), "1"); - UNIT_ASSERT(c.Get("x").empty()); - } - + UNIT_ASSERT_VALUES_EQUAL(c.Get("qw"), "1"); + UNIT_ASSERT(c.Get("x").empty()); + } + Y_UNIT_TEST(TestScanAddAll4) { TCgiParameters c; c.ScanAddAll("ccc=1&aaa=1&ccc=3&bbb&ccc=2"); @@ -191,8 +191,8 @@ Y_UNIT_TEST_SUITE(TCgiParametersTest) { } Y_UNIT_TEST(TestEmpty) { - UNIT_ASSERT(TCgiParameters().Print().empty()); - } + UNIT_ASSERT(TCgiParameters().Print().empty()); + } Y_UNIT_TEST(TestJoinUnescaped) { TCgiParameters c; @@ -218,23 +218,23 @@ Y_UNIT_TEST_SUITE(TCgiParametersTest) { Y_UNIT_TEST(TestPrintAsQuote) { TCgiParameters c = { - std::make_pair("aaa", "value/with/slashes"), + std::make_pair("aaa", "value/with/slashes"), std::make_pair("b/b/b", "value_without_slashes"), - std::make_pair("ccc", "value")}; + std::make_pair("ccc", "value")}; UNIT_ASSERT_VALUES_EQUAL(c.Print(), "aaa=value/with/slashes&b/b/b=value_without_slashes&ccc=value"); UNIT_ASSERT_VALUES_EQUAL(c.QuotedPrint(""), "aaa=value%2Fwith%2Fslashes&b%2Fb%2Fb=value_without_slashes&ccc=value"); } Y_UNIT_TEST(TestPrintAsQuoteEmpty) { - TCgiParameters c = {}; + TCgiParameters c = {}; UNIT_ASSERT_VALUES_EQUAL(c.QuotedPrint(""), ""); } Y_UNIT_TEST(TestPrintAsQuoteEmptyKeyOrValue) { TCgiParameters c = { std::make_pair("", "value/of/empty"), - std::make_pair("key/for/empty", "")}; + std::make_pair("key/for/empty", "")}; UNIT_ASSERT_VALUES_EQUAL(c.Print(), "=value/of/empty&key/for/empty="); UNIT_ASSERT_VALUES_EQUAL(c.QuotedPrint(""), "=value%2Fof%2Fempty&key%2Ffor%2Fempty="); diff --git a/library/cpp/cgiparam/fuzz/main.cpp b/library/cpp/cgiparam/fuzz/main.cpp index 69d82b5f32..f1ff8c4634 100644 --- a/library/cpp/cgiparam/fuzz/main.cpp +++ b/library/cpp/cgiparam/fuzz/main.cpp @@ -1,11 +1,11 @@ #include <library/cpp/cgiparam/cgiparam.h> - -extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { - try { - TCgiParameters(TStringBuf((const char*)data, size)); - } catch (...) { - // ¯\_(ツ)_/¯ - } - - return 0; // Non-zero return values are reserved for future use. -} + +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { + try { + TCgiParameters(TStringBuf((const char*)data, size)); + } catch (...) { + // ¯\_(ツ)_/¯ + } + + return 0; // Non-zero return values are reserved for future use. +} diff --git a/library/cpp/cgiparam/fuzz/ya.make b/library/cpp/cgiparam/fuzz/ya.make index 8fb9d50d3b..594f9c227d 100644 --- a/library/cpp/cgiparam/fuzz/ya.make +++ b/library/cpp/cgiparam/fuzz/ya.make @@ -1,16 +1,16 @@ FUZZ() - + OWNER( pg g:util ) - -SRCS( - main.cpp -) - + +SRCS( + main.cpp +) + PEERDIR( library/cpp/cgiparam ) -END() +END() diff --git a/library/cpp/charset/codepage.cpp b/library/cpp/charset/codepage.cpp index 0431bef31b..816f3fec67 100644 --- a/library/cpp/charset/codepage.cpp +++ b/library/cpp/charset/codepage.cpp @@ -1,45 +1,45 @@ #include "ci_string.h" -#include "wide.h" -#include "recyr.hh" -#include "codepage.h" - -#include <util/string/cast.h> +#include "wide.h" +#include "recyr.hh" +#include "codepage.h" + +#include <util/string/cast.h> #include <util/string/subst.h> -#include <util/string/util.h> +#include <util/string/util.h> #include <util/system/hi_lo.h> -#include <util/system/yassert.h> -#include <util/generic/hash.h> +#include <util/system/yassert.h> +#include <util/generic/hash.h> #include <util/generic/string.h> -#include <util/generic/vector.h> -#include <util/generic/hash_set.h> -#include <util/generic/singleton.h> +#include <util/generic/vector.h> +#include <util/generic/hash_set.h> +#include <util/generic/singleton.h> #include <util/generic/yexception.h> #include <util/memory/pool.h> - -#include <cstring> - -#include <ctype.h> - + +#include <cstring> + +#include <ctype.h> + using namespace NCodepagePrivate; -void Recoder::Create(const CodePage& source, const CodePage& target) { +void Recoder::Create(const CodePage& source, const CodePage& target) { const Encoder* wideTarget = &EncoderByCharset(target.CPEnum); Create(source, wideTarget); } -void Recoder::Create(const CodePage& page, wchar32 (*mapfunc)(wchar32)) { +void Recoder::Create(const CodePage& page, wchar32 (*mapfunc)(wchar32)) { const Encoder* widePage = &EncoderByCharset(page.CPEnum); Create(page, widePage, mapfunc); } -template <class T, class T1> +template <class T, class T1> static inline T1 Apply(T b, T e, T1 to, const Recoder& mapper) { - while (b != e) { - *to++ = mapper.Table[(unsigned char)*b++]; - } - - return to; -} - + while (b != e) { + *to++ = mapper.Table[(unsigned char)*b++]; + } + + return to; +} + template <class T, class T1> static inline T1 Apply(T b, T1 to, const Recoder& mapper) { while (*b != 0) { @@ -49,21 +49,21 @@ static inline T1 Apply(T b, T1 to, const Recoder& mapper) { return to; } -char* CodePage::ToLower(const char* b, const char* e, char* to) const { - return Apply(b, e, to, TCodePageData::rcdr_to_lower[CPEnum]); -} +char* CodePage::ToLower(const char* b, const char* e, char* to) const { + return Apply(b, e, to, TCodePageData::rcdr_to_lower[CPEnum]); +} char* CodePage::ToLower(const char* b, char* to) const { return Apply(b, to, TCodePageData::rcdr_to_lower[CPEnum]); } - -char* CodePage::ToUpper(const char* b, const char* e, char* to) const { + +char* CodePage::ToUpper(const char* b, const char* e, char* to) const { return Apply(b, e, to, TCodePageData::rcdr_to_upper[CPEnum]); -} +} char* CodePage::ToUpper(const char* b, char* to) const { return Apply(b, to, TCodePageData::rcdr_to_upper[CPEnum]); } - -int CodePage::stricmp(const char* dst, const char* src) const { + +int CodePage::stricmp(const char* dst, const char* src) const { unsigned char f, l; do { f = ToLower(*dst++); @@ -86,18 +86,18 @@ int CodePage::strnicmp(const char* dst, const char* src, size_t len) const { static const CodePage UNSUPPORTED_CODEPAGE = { CODES_UNSUPPORTED, - { - "unsupported", - }, + { + "unsupported", + }, {}, nullptr, }; static const CodePage UNKNOWN_CODEPAGE = { CODES_UNKNOWN, - { - "unknown", - }, + { + "unknown", + }, {}, nullptr, }; @@ -122,14 +122,14 @@ NCodepagePrivate::TCodepagesMap::TCodepagesMap() { } } -const NCodepagePrivate::TCodepagesMap& NCodepagePrivate::TCodepagesMap::Instance() { - return *Singleton<NCodepagePrivate::TCodepagesMap>(); -} - +const NCodepagePrivate::TCodepagesMap& NCodepagePrivate::TCodepagesMap::Instance() { + return *Singleton<NCodepagePrivate::TCodepagesMap>(); +} + class TCodePageHash { private: using TData = THashMap<TStringBuf, ECharset, ci_hash, ci_equal_to>; - + TData Data; TMemoryPool Pool; @@ -153,7 +153,7 @@ private: temp = name; SubstGlobal(temp, '-', '_'); AddNameWithCheck(temp, code); - + temp = name; SubstGlobal(temp, '_', '-'); AddNameWithCheck(temp, code); @@ -176,8 +176,8 @@ public: AddName(name, e); AddName(xPrefix + name, e); - } - } + } + } } inline ECharset CharsetByName(TStringBuf name) { @@ -204,7 +204,7 @@ ECharset CharsetByNameOrDie(TStringBuf name) { } template <typename TxChar> -static inline RECODE_RESULT utf8_read_rune_from_unknown_plane(TxChar& rune, size_t& rune_len, const TxChar* s, const TxChar* end) { +static inline RECODE_RESULT utf8_read_rune_from_unknown_plane(TxChar& rune, size_t& rune_len, const TxChar* s, const TxChar* end) { if ((*s & 0xFF00) != 0xF000) { rune_len = 1; rune = *s; @@ -214,37 +214,37 @@ static inline RECODE_RESULT utf8_read_rune_from_unknown_plane(TxChar& rune, size rune_len = 0; size_t _len = UTF8RuneLen((unsigned char)(*s)); - if (s + _len > end) - return RECODE_EOINPUT; //[EOINPUT] - if (_len == 0) - return RECODE_BROKENSYMBOL; //[BROKENSYMBOL] in first byte + if (s + _len > end) + return RECODE_EOINPUT; //[EOINPUT] + if (_len == 0) + return RECODE_BROKENSYMBOL; //[BROKENSYMBOL] in first byte - wchar32 _rune = (ui8)(*s++); //[00000000 0XXXXXXX] + wchar32 _rune = (ui8)(*s++); //[00000000 0XXXXXXX] if (_len > 1) { _rune &= UTF8LeadByteMask(_len); wchar32 ch = *s++; if ((ch & 0xFFC0) != 0xF080) - return RECODE_BROKENSYMBOL; //[BROKENSYMBOL] in second byte + return RECODE_BROKENSYMBOL; //[BROKENSYMBOL] in second byte _rune <<= 6; - _rune |= ch & 0x3F; //[00000XXX XXYYYYYY] + _rune |= ch & 0x3F; //[00000XXX XXYYYYYY] if (_len > 2) { ch = *s++; if ((ch & 0xFFC0) != 0xF080) - return RECODE_BROKENSYMBOL; //[BROKENSYMBOL] in third byte + return RECODE_BROKENSYMBOL; //[BROKENSYMBOL] in third byte _rune <<= 6; - _rune |= ch & 0x3F; //[XXXXYYYY YYZZZZZZ] + _rune |= ch & 0x3F; //[XXXXYYYY YYZZZZZZ] if (_len > 3) { ch = *s; if ((ch & 0xFFC0) != 0xF080) return RECODE_BROKENSYMBOL; //[BROKENSYMBOL] in fourth byte _rune <<= 6; - _rune |= ch & 0x3F; //[XXXYY YYYYZZZZ ZZQQQQQQ] + _rune |= ch & 0x3F; //[XXXYY YYYYZZZZ ZZQQQQQQ] } } } rune_len = _len; if (_rune > Max<TxChar>()) - rune = ' '; // maybe put sequence + rune = ' '; // maybe put sequence else rune = TxChar(_rune); return RECODE_OK; @@ -262,16 +262,16 @@ void DoDecodeUnknownPlane(TxChar* str, TxChar*& ee, const ECharset enc) { } else if (enc == CODES_UTF8) { TxChar* s; TxChar* d; - - for (s = d = str; s < e;) { + + for (s = d = str; s < e;) { size_t l = 0; - + if (utf8_read_rune_from_unknown_plane(*d, l, s, e) == RECODE_OK) { d++, s += l; } else { *d++ = BROKEN_RUNE; ++s; - } + } } e = d; } else if (enc == CODES_UNKNOWN) { @@ -289,7 +289,7 @@ void DoDecodeUnknownPlane(TxChar* str, TxChar*& ee, const ECharset enc) { size_t read = 0; size_t written = 0; - for (; s < e; ++s) { + for (; s < e; ++s) { if (Hi8(Lo16(*s)) == 0xF0) { buf.push_back(Lo8(Lo16(*s))); } else { @@ -318,28 +318,28 @@ void DecodeUnknownPlane(wchar32* str, wchar32*& ee, const ECharset enc) { DoDecodeUnknownPlane(str, ee, enc); } -namespace { +namespace { class THashSetType: public THashSet<TString> { - public: + public: inline void Add(const TString& s) { - insert(s); - } - + insert(s); + } + inline bool Has(const TString& s) const noexcept { - return find(s) != end(); - } - }; -} - + return find(s) != end(); + } + }; +} + class TWindowsPrefixesHashSet: public THashSetType { public: inline TWindowsPrefixesHashSet() { - Add("win"); - Add("wincp"); - Add("window"); + Add("win"); + Add("wincp"); + Add("window"); Add("windowcp"); - Add("windows"); - Add("windowscp"); + Add("windows"); + Add("windowscp"); Add("ansi"); Add("ansicp"); } @@ -364,19 +364,19 @@ public: }; class TLatinToIsoHash: public THashMap<const char*, TString, ci_hash, ci_equal_to> { -public: - inline TLatinToIsoHash() { - insert(value_type("latin1", "iso-8859-1")); - insert(value_type("latin2", "iso-8859-2")); - insert(value_type("latin3", "iso-8859-3")); - insert(value_type("latin4", "iso-8859-4")); - insert(value_type("latin5", "iso-8859-9")); - insert(value_type("latin6", "iso-8859-10")); - insert(value_type("latin7", "iso-8859-13")); - insert(value_type("latin8", "iso-8859-14")); - insert(value_type("latin9", "iso-8859-15")); - insert(value_type("latin10", "iso-8859-16")); - } +public: + inline TLatinToIsoHash() { + insert(value_type("latin1", "iso-8859-1")); + insert(value_type("latin2", "iso-8859-2")); + insert(value_type("latin3", "iso-8859-3")); + insert(value_type("latin4", "iso-8859-4")); + insert(value_type("latin5", "iso-8859-9")); + insert(value_type("latin6", "iso-8859-10")); + insert(value_type("latin7", "iso-8859-13")); + insert(value_type("latin8", "iso-8859-14")); + insert(value_type("latin9", "iso-8859-15")); + insert(value_type("latin10", "iso-8859-16")); + } }; static inline void NormalizeEncodingPrefixes(TString& enc) { @@ -391,14 +391,14 @@ static inline void NormalizeEncodingPrefixes(TString& enc) { } } - if (Singleton<TWindowsPrefixesHashSet>()->Has(prefix)) { + if (Singleton<TWindowsPrefixesHashSet>()->Has(prefix)) { enc.remove(0, preflen); enc.prepend("windows-"); return; } - if (Singleton<TCpPrefixesHashSet>()->Has(prefix)) { - if (enc.length() > preflen + 3 && !strncmp(enc.c_str() + preflen, "125", 3) && isdigit(enc[preflen + 3])) { + if (Singleton<TCpPrefixesHashSet>()->Has(prefix)) { + if (enc.length() > preflen + 3 && !strncmp(enc.c_str() + preflen, "125", 3) && isdigit(enc[preflen + 3])) { enc.remove(0, preflen); enc.prepend("windows-"); return; @@ -408,7 +408,7 @@ static inline void NormalizeEncodingPrefixes(TString& enc) { return; } - if (Singleton<TIsoPrefixesHashSet>()->Has(prefix)) { + if (Singleton<TIsoPrefixesHashSet>()->Has(prefix)) { if (enc.length() == preflen + 1 || enc.length() == preflen + 2) { TString enccopy = enc.substr(preflen); enccopy.prepend("latin"); @@ -428,46 +428,46 @@ static inline void NormalizeEncodingPrefixes(TString& enc) { class TEncodingNamesHashSet: public THashSetType { public: TEncodingNamesHashSet() { - Add("iso-8859-1"); - Add("iso-8859-2"); - Add("iso-8859-3"); - Add("iso-8859-4"); - Add("iso-8859-5"); - Add("iso-8859-6"); - Add("iso-8859-7"); - Add("iso-8859-8"); - Add("iso-8859-8-i"); - Add("iso-8859-9"); - Add("iso-8859-10"); - Add("iso-8859-11"); - Add("iso-8859-12"); - Add("iso-8859-13"); - Add("iso-8859-14"); - Add("iso-8859-15"); - Add("windows-1250"); - Add("windows-1251"); - Add("windows-1252"); - Add("windows-1253"); - Add("windows-1254"); - Add("windows-1255"); - Add("windows-1256"); - Add("windows-1257"); - Add("windows-1258"); - Add("windows-874"); - Add("iso-2022-jp"); - Add("euc-jp"); - Add("shift-jis"); - Add("shiftjis"); - Add("iso-2022-kr"); - Add("euc-kr"); - Add("gb-2312"); - Add("gb2312"); - Add("gb-18030"); - Add("gb18030"); - Add("gbk"); - Add("big5"); - Add("tis-620"); - Add("tis620"); + Add("iso-8859-1"); + Add("iso-8859-2"); + Add("iso-8859-3"); + Add("iso-8859-4"); + Add("iso-8859-5"); + Add("iso-8859-6"); + Add("iso-8859-7"); + Add("iso-8859-8"); + Add("iso-8859-8-i"); + Add("iso-8859-9"); + Add("iso-8859-10"); + Add("iso-8859-11"); + Add("iso-8859-12"); + Add("iso-8859-13"); + Add("iso-8859-14"); + Add("iso-8859-15"); + Add("windows-1250"); + Add("windows-1251"); + Add("windows-1252"); + Add("windows-1253"); + Add("windows-1254"); + Add("windows-1255"); + Add("windows-1256"); + Add("windows-1257"); + Add("windows-1258"); + Add("windows-874"); + Add("iso-2022-jp"); + Add("euc-jp"); + Add("shift-jis"); + Add("shiftjis"); + Add("iso-2022-kr"); + Add("euc-kr"); + Add("gb-2312"); + Add("gb2312"); + Add("gb-18030"); + Add("gb18030"); + Add("gbk"); + Add("big5"); + Add("tis-620"); + Add("tis620"); } }; @@ -494,7 +494,7 @@ ECharset EncodingHintByName(const char* encname) { // Do some normalization TString enc(encname, lastpos - encname + 1); enc.to_lower(); - for (char* p = enc.begin(); p != enc.end(); ++p) { + for (char* p = enc.begin(); p != enc.end(); ++p) { if (*p == ' ' || *p == '=' || *p == '_') *p = '-'; } @@ -505,7 +505,7 @@ ECharset EncodingHintByName(const char* encname) { if (hint != CODES_UNKNOWN) return hint; - if (Singleton<TEncodingNamesHashSet>()->Has(enc)) + if (Singleton<TEncodingNamesHashSet>()->Has(enc)) return CODES_UNSUPPORTED; return CODES_UNKNOWN; } diff --git a/library/cpp/charset/codepage.h b/library/cpp/charset/codepage.h index 30a02a4610..2911174dce 100644 --- a/library/cpp/charset/codepage.h +++ b/library/cpp/charset/codepage.h @@ -1,6 +1,6 @@ #pragma once -#include "doccodes.h" +#include "doccodes.h" #include <util/charset/recode_result.h> #include <util/charset/unidata.h> // all wchar32 functions @@ -8,11 +8,11 @@ #include <util/generic/string.h> #include <util/generic/ylimits.h> #include <util/generic/yexception.h> -#include <util/system/yassert.h> -#include <util/system/defaults.h> - -#include <cctype> - +#include <util/system/yassert.h> +#include <util/system/defaults.h> + +#include <cctype> + struct CodePage; struct Recoder; struct Encoder; @@ -21,10 +21,10 @@ struct Encoder; * struct CodePage * \*****************************************************************/ struct CodePage { - ECharset CPEnum; // int MIBEnum; - const char* Names[30]; // name[0] -- preferred mime-name - wchar32 unicode[256]; - const char* DefaultChar; //[CCL_NUM] + ECharset CPEnum; // int MIBEnum; + const char* Names[30]; // name[0] -- preferred mime-name + wchar32 unicode[256]; + const char* DefaultChar; //[CCL_NUM] bool IsLower(unsigned char ch) const { return ::IsLower(unicode[ch]); @@ -38,7 +38,7 @@ struct CodePage { bool IsDigit(unsigned char ch) const { return ::IsDigit(unicode[ch]); } - bool IsXdigit(unsigned char ch) const { + bool IsXdigit(unsigned char ch) const { return ::IsXdigit(unicode[ch]); } bool IsAlnum(unsigned char ch) const { @@ -62,18 +62,18 @@ struct CodePage { bool IsComposed(unsigned char ch) const { return ::IsComposed(unicode[ch]); } - + // return pointer to char after the last char - char* ToLower(const char* begin, const char* end, char* to) const; + char* ToLower(const char* begin, const char* end, char* to) const; char* ToLower(const char* begin, char* to) const; - // return pointer to char after the last char - char* ToUpper(const char* begin, const char* end, char* to) const; + // return pointer to char after the last char + char* ToUpper(const char* begin, const char* end, char* to) const; char* ToUpper(const char* begin, char* to) const; - - int stricmp(const char* s1, const char* s2) const; - int strnicmp(const char* s1, const char* s2, size_t len) const; - + + int stricmp(const char* s1, const char* s2) const; + int strnicmp(const char* s1, const char* s2, size_t len) const; + inline unsigned char ToUpper(unsigned char ch) const; inline unsigned char ToLower(unsigned char ch) const; inline unsigned char ToTitle(unsigned char ch) const; @@ -131,18 +131,18 @@ namespace NCodepagePrivate { return GetPrivate(e)->Names[0]; } - static const TCodepagesMap& Instance(); - + static const TCodepagesMap& Instance(); + friend class ::TCodePageHash; }; inline bool NativeCodepage(ECharset e) { - return ::NCodepagePrivate::TCodepagesMap::Instance().NativeCodepage(e); + return ::NCodepagePrivate::TCodepagesMap::Instance().NativeCodepage(e); } } inline bool SingleByteCodepage(ECharset e) { - return ::NCodepagePrivate::TCodepagesMap::Instance().SingleByteCodepage(e); + return ::NCodepagePrivate::TCodepagesMap::Instance().SingleByteCodepage(e); } inline bool ValidCodepage(ECharset e) { @@ -150,7 +150,7 @@ inline bool ValidCodepage(ECharset e) { } inline const CodePage* CodePageByCharset(ECharset e) { - return ::NCodepagePrivate::TCodepagesMap::Instance().Get(e); + return ::NCodepagePrivate::TCodepagesMap::Instance().Get(e); } ECharset CharsetByName(TStringBuf name); @@ -163,12 +163,12 @@ inline ECharset CharsetByCodePage(const CodePage* CP) { } inline const char* NameByCharset(ECharset e) { - return ::NCodepagePrivate::TCodepagesMap::Instance().NameByCharset(e); + return ::NCodepagePrivate::TCodepagesMap::Instance().NameByCharset(e); } inline const char* NameByCharsetSafe(ECharset e) { if (CODES_UNKNOWN < e && e < CODES_MAX) - return ::NCodepagePrivate::TCodepagesMap::Instance().NameByCharset(e); + return ::NCodepagePrivate::TCodepagesMap::Instance().NameByCharset(e); else ythrow yexception() << "unknown encoding: " << (int)e; } @@ -194,21 +194,21 @@ struct Encoder { char* Table[256]; const char* DefaultChar; - inline char Code(wchar32 ch) const { + inline char Code(wchar32 ch) const { if (ch > 0xFFFF) return 0; - return (unsigned char)Table[(ch >> 8) & 255][ch & 255]; + return (unsigned char)Table[(ch >> 8) & 255][ch & 255]; } - inline char Tr(wchar32 ch) const { + inline char Tr(wchar32 ch) const { char code = Code(ch); if (code == 0 && ch != 0) - code = DefaultChar[NUnicode::CharType(ch)]; + code = DefaultChar[NUnicode::CharType(ch)]; Y_ASSERT(code != 0 || ch == 0); return code; } - inline unsigned char operator[](wchar32 ch) const { + inline unsigned char operator[](wchar32 ch) const { return Tr(ch); } @@ -223,25 +223,25 @@ struct Encoder { struct Recoder { unsigned char Table[257]; - void Create(const CodePage& source, const CodePage& target); - void Create(const CodePage& source, const Encoder* wideTarget); + void Create(const CodePage& source, const CodePage& target); + void Create(const CodePage& source, const Encoder* wideTarget); - void Create(const CodePage& page, wchar32 (*mapper)(wchar32)); - void Create(const CodePage& page, const Encoder* widePage, wchar32 (*mapper)(wchar32)); + void Create(const CodePage& page, wchar32 (*mapper)(wchar32)); + void Create(const CodePage& page, const Encoder* widePage, wchar32 (*mapper)(wchar32)); - inline unsigned char Tr(unsigned char c) const { + inline unsigned char Tr(unsigned char c) const { return Table[c]; } - inline unsigned char operator[](unsigned char c) const { + inline unsigned char operator[](unsigned char c) const { return Table[c]; } - void Tr(const char* in, char* out, size_t len) const; - void Tr(const char* in, char* out) const; - void Tr(char* in_out, size_t len) const; - void Tr(char* in_out) const; + void Tr(const char* in, char* out, size_t len) const; + void Tr(const char* in, char* out) const; + void Tr(char* in_out, size_t len) const; + void Tr(char* in_out) const; }; -extern const struct Encoder& WideCharToYandex; +extern const struct Encoder& WideCharToYandex; const Encoder& EncoderByCharset(ECharset enc); @@ -255,7 +255,7 @@ namespace NCodepagePrivate { static const Recoder rcdr_to_lower[]; static const Recoder rcdr_to_upper[]; static const Recoder rcdr_to_title[]; - + static const Encoder* const EncodeTo[]; friend struct ::CodePage; @@ -264,7 +264,7 @@ namespace NCodepagePrivate { friend RECODE_RESULT _recodeFromYandex(ECharset, const char*, char*, size_t, size_t, size_t&, size_t&); friend const Encoder& ::EncoderByCharset(ECharset enc); }; -} +} inline const Encoder& EncoderByCharset(ECharset enc) { if (!SingleByteCodepage(enc)) { @@ -319,6 +319,6 @@ inline TString ToTitle(TString s, const CodePage& cp, size_t pos = 0, size_t n = return i == pos ? cp.ToTitle(c) : cp.ToLower(c); }, pos, - n); + n); return s; } diff --git a/library/cpp/charset/codepage_ut.cpp b/library/cpp/charset/codepage_ut.cpp index c3ac3ac478..1a572cac44 100644 --- a/library/cpp/charset/codepage_ut.cpp +++ b/library/cpp/charset/codepage_ut.cpp @@ -1,47 +1,47 @@ #include "codepage.h" #include "recyr.hh" -#include "wide.h" - +#include "wide.h" + #include <library/cpp/testing/unittest/registar.h> - + #include <util/charset/utf8.h> #include <util/system/yassert.h> -#if defined(_MSC_VER) -#pragma warning(disable : 4309) /*truncation of constant value*/ +#if defined(_MSC_VER) +#pragma warning(disable : 4309) /*truncation of constant value*/ #endif namespace { const char yandexUpperCase[] = - "\x81\x82\x83\x84\x85\x86\x87" - "\x8E" - "\xA1\xA2\xA3\xA4\xA5\xA6" - "\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF" + "\x81\x82\x83\x84\x85\x86\x87" + "\x8E" + "\xA1\xA2\xA3\xA4\xA5\xA6" + "\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF" "\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA\xCB\xCC\xCD\xCE\xCF" "\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF"; const char yandexLowerCase[] = - "\x91\x92\x93\x94\x95\x96\x97" - "\x9E" - "\xB1\xB2\xB3\xB4\xB5\xB6" - "\xB8\xB9\xBA\xBB\xBC\xBD\xBE\xBF" + "\x91\x92\x93\x94\x95\x96\x97" + "\x9E" + "\xB1\xB2\xB3\xB4\xB5\xB6" + "\xB8\xB9\xBA\xBB\xBC\xBD\xBE\xBF" "\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF" "\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7\xF8\xF9\xFA\xFB\xFC\xFD\xFE\xFF"; } -class TCodepageTest: public TTestBase { +class TCodepageTest: public TTestBase { private: UNIT_TEST_SUITE(TCodepageTest); - UNIT_TEST(TestUTF); - UNIT_TEST(TestUTFFromUnknownPlane); - UNIT_TEST(TestBrokenMultibyte); - UNIT_TEST(TestSurrogatePairs); - UNIT_TEST(TestEncodingHints); - UNIT_TEST(TestToLower); - UNIT_TEST(TestToUpper); - UNIT_TEST(TestUpperLower); - UNIT_TEST(TestBrokenRune); - UNIT_TEST(TestCanEncode); + UNIT_TEST(TestUTF); + UNIT_TEST(TestUTFFromUnknownPlane); + UNIT_TEST(TestBrokenMultibyte); + UNIT_TEST(TestSurrogatePairs); + UNIT_TEST(TestEncodingHints); + UNIT_TEST(TestToLower); + UNIT_TEST(TestToUpper); + UNIT_TEST(TestUpperLower); + UNIT_TEST(TestBrokenRune); + UNIT_TEST(TestCanEncode); UNIT_TEST_SUITE_END(); public: @@ -55,18 +55,18 @@ public: void TestCanEncode(); - inline void TestUpperLower() { + inline void TestUpperLower() { const CodePage* cp = CodePageByCharset(CODES_ASCII); - char tmp[100]; - + char tmp[100]; + TStringBuf s = "abcde"; - - TStringBuf upper(tmp, cp->ToUpper(s.begin(), s.end(), tmp)); + + TStringBuf upper(tmp, cp->ToUpper(s.begin(), s.end(), tmp)); UNIT_ASSERT_VALUES_EQUAL(upper, TStringBuf("ABCDE")); - - TStringBuf lower(tmp, cp->ToLower(upper.begin(), upper.end(), tmp)); + + TStringBuf lower(tmp, cp->ToLower(upper.begin(), upper.end(), tmp)); UNIT_ASSERT_VALUES_EQUAL(lower, TStringBuf("abcde")); - } + } void TestBrokenRune() { UNIT_ASSERT_VALUES_EQUAL(BROKEN_RUNE, 0xFFFDu); @@ -78,7 +78,7 @@ UNIT_TEST_SUITE_REGISTRATION(TCodepageTest); void TCodepageTest::TestUTF() { for (wchar32 i = 0; i <= 0x10FFFF; i++) { unsigned char buffer[32]; - Zero(buffer); + Zero(buffer); size_t rune_len; size_t ref_len = 0; @@ -120,7 +120,7 @@ void TCodepageTest::TestUTF() { UNIT_ASSERT(res == RECODE_BROKENSYMBOL); } } - const char* badStrings[] = { + const char* badStrings[] = { "\xfe", "\xff", "\xcc\xc0", @@ -153,7 +153,7 @@ void TCodepageTest::TestUTF() { }; for (size_t i = 0; i < Y_ARRAY_SIZE(badStrings); ++i) { wchar32 rune; - const ui8* p = (const ui8*)badStrings[i]; + const ui8* p = (const ui8*)badStrings[i]; size_t len; RECODE_RESULT res = SafeReadUTF8Char(rune, len, p, p + strlen(badStrings[i])); UNIT_ASSERT(res == RECODE_BROKENSYMBOL); @@ -174,17 +174,17 @@ void TCodepageTest::TestBrokenMultibyte() { UNIT_ASSERT(nread == 1); UNIT_ASSERT(nwritten == 0); - const char bigSample[] = {'\xC3', '\x87', '\xC3', '\x8E', '\xC2', '\xB0', '\xC3', '\x85', '\xC3', '\x85', '\xC3', '\xB8'}; + const char bigSample[] = {'\xC3', '\x87', '\xC3', '\x8E', '\xC2', '\xB0', '\xC3', '\x85', '\xC3', '\x85', '\xC3', '\xB8'}; res = RecodeToUnicode(cp, bigSample, recodeResult, Y_ARRAY_SIZE(bigSample), Y_ARRAY_SIZE(recodeResult), nread, nwritten); UNIT_ASSERT(res == RECODE_OK); UNIT_ASSERT(nread == Y_ARRAY_SIZE(bigSample)); } void TCodepageTest::TestUTFFromUnknownPlane() { - static const wchar32 sampletext[] = {0x61, 0x62, 0x63, 0x20, - 0x430, 0x431, 0x432, 0x20, - 0x1001, 0x1002, 0x1003, 0x20, - 0x10001, 0x10002, 0x10003}; + static const wchar32 sampletext[] = {0x61, 0x62, 0x63, 0x20, + 0x430, 0x431, 0x432, 0x20, + 0x1001, 0x1002, 0x1003, 0x20, + 0x10001, 0x10002, 0x10003}; static const size_t BUFFER_SIZE = 1024; char bytebuffer[BUFFER_SIZE]; @@ -192,17 +192,17 @@ void TCodepageTest::TestUTFFromUnknownPlane() { size_t readchars = 0; size_t writtenbytes = 0; size_t samplelen = Y_ARRAY_SIZE(sampletext); + + RECODE_RESULT res = RecodeFromUnicode(CODES_UTF8, sampletext, bytebuffer, samplelen, BUFFER_SIZE, readchars, writtenbytes); - RECODE_RESULT res = RecodeFromUnicode(CODES_UTF8, sampletext, bytebuffer, samplelen, BUFFER_SIZE, readchars, writtenbytes); - - UNIT_ASSERT(res == RECODE_OK); - UNIT_ASSERT(samplelen == readchars); + UNIT_ASSERT(res == RECODE_OK); + UNIT_ASSERT(samplelen == readchars); size_t writtenbytes2 = 0; char bytebuffer2[BUFFER_SIZE]; for (size_t i = 0; i != samplelen; ++i) { size_t nwr = 0; - const int res = RecodeFromUnicode(CODES_UTF8, sampletext[i], bytebuffer2 + writtenbytes2, BUFFER_SIZE - writtenbytes2, nwr); + const int res = RecodeFromUnicode(CODES_UTF8, sampletext[i], bytebuffer2 + writtenbytes2, BUFFER_SIZE - writtenbytes2, nwr); UNIT_ASSERT_VALUES_EQUAL(res, int(RECODE_OK)); writtenbytes2 += nwr; UNIT_ASSERT(BUFFER_SIZE > writtenbytes2); @@ -213,43 +213,43 @@ void TCodepageTest::TestUTFFromUnknownPlane() { size_t readbytes = 0; size_t writtenchars = 0; - res = RecodeToUnicode(CODES_UNKNOWNPLANE, bytebuffer, charbuffer, writtenbytes, BUFFER_SIZE, readbytes, writtenchars); + res = RecodeToUnicode(CODES_UNKNOWNPLANE, bytebuffer, charbuffer, writtenbytes, BUFFER_SIZE, readbytes, writtenchars); - UNIT_ASSERT(res == RECODE_OK); - UNIT_ASSERT(readbytes == writtenbytes); + UNIT_ASSERT(res == RECODE_OK); + UNIT_ASSERT(readbytes == writtenbytes); wchar32* charbufferend = charbuffer + writtenchars; - DecodeUnknownPlane(charbuffer, charbufferend, CODES_UTF8); + DecodeUnknownPlane(charbuffer, charbufferend, CODES_UTF8); - UNIT_ASSERT(charbufferend == charbuffer + samplelen); - for (size_t i = 0; i < samplelen; ++i) - UNIT_ASSERT(sampletext[i] == charbuffer[i]); + UNIT_ASSERT(charbufferend == charbuffer + samplelen); + for (size_t i = 0; i < samplelen; ++i) + UNIT_ASSERT(sampletext[i] == charbuffer[i]); // Now, concatenate the thing with an explicit character and retest - res = RecodeToUnicode(CODES_UNKNOWNPLANE, bytebuffer, charbuffer, writtenbytes, BUFFER_SIZE, readbytes, writtenchars); - UNIT_ASSERT(res == RECODE_OK); - UNIT_ASSERT(readbytes == writtenbytes); + res = RecodeToUnicode(CODES_UNKNOWNPLANE, bytebuffer, charbuffer, writtenbytes, BUFFER_SIZE, readbytes, writtenchars); + UNIT_ASSERT(res == RECODE_OK); + UNIT_ASSERT(readbytes == writtenbytes); charbuffer[writtenchars] = 0x1234; size_t morewrittenchars = 0; - res = RecodeToUnicode(CODES_UNKNOWNPLANE, bytebuffer, charbuffer + writtenchars + 1, writtenbytes, BUFFER_SIZE, readbytes, morewrittenchars); - UNIT_ASSERT(res == RECODE_OK); - UNIT_ASSERT(readbytes == writtenbytes); - UNIT_ASSERT(writtenchars == morewrittenchars); + res = RecodeToUnicode(CODES_UNKNOWNPLANE, bytebuffer, charbuffer + writtenchars + 1, writtenbytes, BUFFER_SIZE, readbytes, morewrittenchars); + UNIT_ASSERT(res == RECODE_OK); + UNIT_ASSERT(readbytes == writtenbytes); + UNIT_ASSERT(writtenchars == morewrittenchars); charbuffer[2 * writtenchars + 1] = 0x5678; charbufferend = charbuffer + 2 * writtenchars + 2; - DecodeUnknownPlane(charbuffer, charbufferend, CODES_UTF8); + DecodeUnknownPlane(charbuffer, charbufferend, CODES_UTF8); - UNIT_ASSERT(charbufferend == charbuffer + 2 * samplelen + 2); + UNIT_ASSERT(charbufferend == charbuffer + 2 * samplelen + 2); for (size_t i = 0; i < samplelen; ++i) { - UNIT_ASSERT(sampletext[i] == charbuffer[i]); - UNIT_ASSERT(sampletext[i] == charbuffer[samplelen + 1 + i]); + UNIT_ASSERT(sampletext[i] == charbuffer[i]); + UNIT_ASSERT(sampletext[i] == charbuffer[samplelen + 1 + i]); } - UNIT_ASSERT(0x1234 == charbuffer[samplelen]); - UNIT_ASSERT(0x5678 == charbuffer[2 * samplelen + 1]); + UNIT_ASSERT(0x1234 == charbuffer[samplelen]); + UNIT_ASSERT(0x5678 == charbuffer[2 * samplelen + 1]); // test TChar version // bytebuffer of len writtenbytes contains sampletext of len samplelen chars in utf8 @@ -261,7 +261,7 @@ void TCodepageTest::TestUTFFromUnknownPlane() { for (size_t i = 0; i < wtr.size(); ++i) { if (sampletext[i] >= 0x10000) { UNIT_ASSERT_VALUES_EQUAL(wtr[i], ' '); - } else { + } else { UNIT_ASSERT_VALUES_EQUAL(wtr[i], sampletext[i]); } } @@ -290,11 +290,11 @@ static void TestSurrogates(const char* str, const wchar16* wide, size_t wideSize void TCodepageTest::TestSurrogatePairs() { const char* utf8NonBMP = "\xf4\x80\x89\x84\xf4\x80\x89\x87\xf4\x80\x88\xba"; - wchar16 wNonBMPDummy[] = {0xDBC0, 0xDE44, 0xDBC0, 0xDE47, 0xDBC0, 0xDE3A}; + wchar16 wNonBMPDummy[] = {0xDBC0, 0xDE44, 0xDBC0, 0xDE47, 0xDBC0, 0xDE3A}; TestSurrogates(utf8NonBMP, wNonBMPDummy, Y_ARRAY_SIZE(wNonBMPDummy)); const char* utf8NonBMP2 = "ab\xf4\x80\x89\x87n"; - wchar16 wNonBMPDummy2[] = {'a', 'b', 0xDBC0, 0xDE47, 'n'}; + wchar16 wNonBMPDummy2[] = {'a', 'b', 0xDBC0, 0xDE47, 'n'}; TestSurrogates(utf8NonBMP2, wNonBMPDummy2, Y_ARRAY_SIZE(wNonBMPDummy2)); } @@ -356,7 +356,7 @@ static void TestCanEncodeEach(const TWtringBuf& text, ECharset encoding, bool ex for (size_t i = 0; i < text.size(); ++i) { if (CanBeEncoded(text.SubStr(i, 1), encoding) != expectedResult) ythrow yexception() << "assertion failed: encoding " << NameByCharset(encoding) - << " on '" << text.SubStr(i, 1) << "' (expected " << expectedResult << ")"; + << " on '" << text.SubStr(i, 1) << "' (expected " << expectedResult << ")"; } // whole text UNIT_ASSERT_EQUAL(CanBeEncoded(text, encoding), expectedResult); diff --git a/library/cpp/charset/cp_encrec.cpp b/library/cpp/charset/cp_encrec.cpp index e4570cd628..aa68278a04 100644 --- a/library/cpp/charset/cp_encrec.cpp +++ b/library/cpp/charset/cp_encrec.cpp @@ -1,5 +1,5 @@ -#include "codepage.h" - +#include "codepage.h" + #include <util/stream/output.h> void Encoder::Tr(const wchar32* in, char* out, size_t len) const { @@ -13,14 +13,14 @@ void Encoder::Tr(const wchar32* in, char* out) const { } while (*in++); } -void Recoder::Create(const CodePage& source, const Encoder* wideTarget) { - for (size_t i = 0; i != 256; ++i) { +void Recoder::Create(const CodePage& source, const Encoder* wideTarget) { + for (size_t i = 0; i != 256; ++i) { Table[i] = wideTarget->Tr(source.unicode[i]); Y_ASSERT(Table[i] != 0 || i == 0); } } -void Recoder::Create(const CodePage& page, const Encoder* widePage, wchar32 (*mapfunc)(wchar32)) { +void Recoder::Create(const CodePage& page, const Encoder* widePage, wchar32 (*mapfunc)(wchar32)) { for (size_t i = 0; i != 256; ++i) { char c = widePage->Code((*mapfunc)(page.unicode[i])); Table[i] = (c == 0 && i != 0) ? (unsigned char)i : (unsigned char)c; diff --git a/library/cpp/charset/doccodes.cpp b/library/cpp/charset/doccodes.cpp index 1fc17a3275..e0384a7f88 100644 --- a/library/cpp/charset/doccodes.cpp +++ b/library/cpp/charset/doccodes.cpp @@ -1 +1 @@ -#include "doccodes.h" +#include "doccodes.h" diff --git a/library/cpp/charset/doccodes.h b/library/cpp/charset/doccodes.h index 75c87adf9e..0aa7eb2d2b 100644 --- a/library/cpp/charset/doccodes.h +++ b/library/cpp/charset/doccodes.h @@ -1,45 +1,45 @@ #pragma once -enum ECharset { +enum ECharset { CODES_UNSUPPORTED = -2, // valid but unsupported encoding - CODES_UNKNOWN = -1, // invalid or unspecified encoding - CODES_WIN, // [ 0] WINDOWS_1251 Windows - CODES_KOI8, // [ 1] KOI8_U Koi8-u - CODES_ALT, // [ 2] IBM_866 MS DOS, alternative - CODES_MAC, // [ 3] MAC_CYRILLIC Macintosh - CODES_MAIN, // [ 4] ISO_LATIN_CYRILLIC Main - CODES_ASCII, // [ 5] WINDOWS_1252 Latin 1 - CODES_RESERVED_3, // reserved code: use it for new encodings before adding them to the end of the list - CODES_WIN_EAST, // [ 7] WINDOWS_1250 WIN PL - CODES_ISO_EAST, // [ 8] ISO_8859_2 ISO PL + CODES_UNKNOWN = -1, // invalid or unspecified encoding + CODES_WIN, // [ 0] WINDOWS_1251 Windows + CODES_KOI8, // [ 1] KOI8_U Koi8-u + CODES_ALT, // [ 2] IBM_866 MS DOS, alternative + CODES_MAC, // [ 3] MAC_CYRILLIC Macintosh + CODES_MAIN, // [ 4] ISO_LATIN_CYRILLIC Main + CODES_ASCII, // [ 5] WINDOWS_1252 Latin 1 + CODES_RESERVED_3, // reserved code: use it for new encodings before adding them to the end of the list + CODES_WIN_EAST, // [ 7] WINDOWS_1250 WIN PL + CODES_ISO_EAST, // [ 8] ISO_8859_2 ISO PL // our superset of subset of windows-1251 - CODES_YANDEX, // [ 9] YANDEX - CODES_UTF_16BE, // [10] UTF_16BE - CODES_UTF_16LE, // [11] UTF_16LE + CODES_YANDEX, // [ 9] YANDEX + CODES_UTF_16BE, // [10] UTF_16BE + CODES_UTF_16LE, // [11] UTF_16LE // missing standard codepages - CODES_IBM855, // [12] IBM_855 - CODES_UTF8, // [13] UTF8 - CODES_UNKNOWNPLANE, // [14] Unrecognized characters are mapped into the PUA: U+F000..U+F0FF + CODES_IBM855, // [12] IBM_855 + CODES_UTF8, // [13] UTF8 + CODES_UNKNOWNPLANE, // [14] Unrecognized characters are mapped into the PUA: U+F000..U+F0FF - CODES_KAZWIN, // [15] WINDOWS_1251_K Kazakh version of Windows-1251 - CODES_TATWIN, // [16] WINDOWS_1251_T Tatarian version of Windows-1251 - CODES_ARMSCII, // [17] Armenian ASCII - CODES_GEO_ITA, // [18] Academy of Sciences Georgian - CODES_GEO_PS, // [19] Georgian Parliament - CODES_ISO_8859_3, // [20] Latin-3: Turkish, Maltese and Esperanto - CODES_ISO_8859_4, // [21] Latin-4: Estonian, Latvian, Lithuanian, Greenlandic, Sami - CODES_ISO_8859_6, // [22] Latin/Arabic: Arabic - CODES_ISO_8859_7, // [23] Latin/Greek: Greek - CODES_ISO_8859_8, // [24] Latin/Hebrew: Hebrew - CODES_ISO_8859_9, // [25] Latin-5 or Turkish: Turkish - CODES_ISO_8859_13, // [26] Latin-7 or Baltic Rim: Baltic languages - CODES_ISO_8859_15, // [27] Latin-9: Western European languages - CODES_ISO_8859_16, // [28] Latin-10: South-Eastern European languages - CODES_WINDOWS_1253, // [29] for Greek - CODES_WINDOWS_1254, // [30] for Turkish - CODES_WINDOWS_1255, // [31] for Hebrew - CODES_WINDOWS_1256, // [32] for Arabic - CODES_WINDOWS_1257, // [33] for Estonian, Latvian and Lithuanian + CODES_KAZWIN, // [15] WINDOWS_1251_K Kazakh version of Windows-1251 + CODES_TATWIN, // [16] WINDOWS_1251_T Tatarian version of Windows-1251 + CODES_ARMSCII, // [17] Armenian ASCII + CODES_GEO_ITA, // [18] Academy of Sciences Georgian + CODES_GEO_PS, // [19] Georgian Parliament + CODES_ISO_8859_3, // [20] Latin-3: Turkish, Maltese and Esperanto + CODES_ISO_8859_4, // [21] Latin-4: Estonian, Latvian, Lithuanian, Greenlandic, Sami + CODES_ISO_8859_6, // [22] Latin/Arabic: Arabic + CODES_ISO_8859_7, // [23] Latin/Greek: Greek + CODES_ISO_8859_8, // [24] Latin/Hebrew: Hebrew + CODES_ISO_8859_9, // [25] Latin-5 or Turkish: Turkish + CODES_ISO_8859_13, // [26] Latin-7 or Baltic Rim: Baltic languages + CODES_ISO_8859_15, // [27] Latin-9: Western European languages + CODES_ISO_8859_16, // [28] Latin-10: South-Eastern European languages + CODES_WINDOWS_1253, // [29] for Greek + CODES_WINDOWS_1254, // [30] for Turkish + CODES_WINDOWS_1255, // [31] for Hebrew + CODES_WINDOWS_1256, // [32] for Arabic + CODES_WINDOWS_1257, // [33] for Estonian, Latvian and Lithuanian // these codes are all the other 8bit codes known by libiconv // they follow in alphanumeric order diff --git a/library/cpp/charset/iconv.cpp b/library/cpp/charset/iconv.cpp index df43471470..605d0699ef 100644 --- a/library/cpp/charset/iconv.cpp +++ b/library/cpp/charset/iconv.cpp @@ -1,94 +1,94 @@ -#include "iconv.h" - -#include <contrib/libs/libiconv/iconv.h> - -using namespace NICONVPrivate; - -TDescriptor::TDescriptor(const char* from, const char* to) - : Descriptor_(libiconv_open(to, from)) - , From_(from) - , To_(to) -{ - if (!Invalid()) { - int temp = 1; - - libiconvctl(Descriptor_, ICONV_SET_DISCARD_ILSEQ, &temp); - } -} - -TDescriptor::~TDescriptor() { - if (!Invalid()) { - libiconv_close(Descriptor_); - } -} - -size_t NICONVPrivate::RecodeImpl(const TDescriptor& descriptor, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { - Y_ASSERT(!descriptor.Invalid()); - Y_ASSERT(in); - Y_ASSERT(out); - +#include "iconv.h" + +#include <contrib/libs/libiconv/iconv.h> + +using namespace NICONVPrivate; + +TDescriptor::TDescriptor(const char* from, const char* to) + : Descriptor_(libiconv_open(to, from)) + , From_(from) + , To_(to) +{ + if (!Invalid()) { + int temp = 1; + + libiconvctl(Descriptor_, ICONV_SET_DISCARD_ILSEQ, &temp); + } +} + +TDescriptor::~TDescriptor() { + if (!Invalid()) { + libiconv_close(Descriptor_); + } +} + +size_t NICONVPrivate::RecodeImpl(const TDescriptor& descriptor, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { + Y_ASSERT(!descriptor.Invalid()); + Y_ASSERT(in); + Y_ASSERT(out); + char* inPtr = const_cast<char*>(in); - char* outPtr = out; - size_t inSizeMod = inSize; - size_t outSizeMod = outSize; - size_t res = libiconv(descriptor.Get(), &inPtr, &inSizeMod, &outPtr, &outSizeMod); - - read = inSize - inSizeMod; - written = outSize - outSizeMod; - - return res; -} - -void NICONVPrivate::DoRecode(const TDescriptor& descriptor, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { - if (descriptor.Invalid()) { - ythrow yexception() << "Can not convert from " << descriptor.From() << " to " << descriptor.To(); - } - - size_t res = RecodeImpl(descriptor, in, out, inSize, outSize, read, written); - - if (res == static_cast<size_t>(-1)) { - switch (errno) { - case EILSEQ: - read = inSize; - break; - - case EINVAL: - read = inSize; - break; - - case E2BIG: - ythrow yexception() << "Iconv error: output buffer is too small"; - - default: - ythrow yexception() << "Unknown iconv error"; - } - } -} - -RECODE_RESULT NICONVPrivate::DoRecodeNoThrow(const TDescriptor& descriptor, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { - if (descriptor.Invalid()) { - return RECODE_ERROR; - } - - size_t res = RecodeImpl(descriptor, in, out, inSize, outSize, read, written); - - if (res == static_cast<size_t>(-1)) { - switch (errno) { - case EILSEQ: - read = inSize; - break; - - case EINVAL: - read = inSize; - break; - - case E2BIG: - return RECODE_EOOUTPUT; - - default: - return RECODE_ERROR; - } - } - - return RECODE_OK; -} + char* outPtr = out; + size_t inSizeMod = inSize; + size_t outSizeMod = outSize; + size_t res = libiconv(descriptor.Get(), &inPtr, &inSizeMod, &outPtr, &outSizeMod); + + read = inSize - inSizeMod; + written = outSize - outSizeMod; + + return res; +} + +void NICONVPrivate::DoRecode(const TDescriptor& descriptor, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { + if (descriptor.Invalid()) { + ythrow yexception() << "Can not convert from " << descriptor.From() << " to " << descriptor.To(); + } + + size_t res = RecodeImpl(descriptor, in, out, inSize, outSize, read, written); + + if (res == static_cast<size_t>(-1)) { + switch (errno) { + case EILSEQ: + read = inSize; + break; + + case EINVAL: + read = inSize; + break; + + case E2BIG: + ythrow yexception() << "Iconv error: output buffer is too small"; + + default: + ythrow yexception() << "Unknown iconv error"; + } + } +} + +RECODE_RESULT NICONVPrivate::DoRecodeNoThrow(const TDescriptor& descriptor, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { + if (descriptor.Invalid()) { + return RECODE_ERROR; + } + + size_t res = RecodeImpl(descriptor, in, out, inSize, outSize, read, written); + + if (res == static_cast<size_t>(-1)) { + switch (errno) { + case EILSEQ: + read = inSize; + break; + + case EINVAL: + read = inSize; + break; + + case E2BIG: + return RECODE_EOOUTPUT; + + default: + return RECODE_ERROR; + } + } + + return RECODE_OK; +} diff --git a/library/cpp/charset/iconv.h b/library/cpp/charset/iconv.h index ac13539347..58188bb33d 100644 --- a/library/cpp/charset/iconv.h +++ b/library/cpp/charset/iconv.h @@ -10,66 +10,66 @@ namespace NICONVPrivate { inline const char* CharsetName(ECharset code) { return NameByCharset(code); } - + inline const char* CharsetName(const char* code) { return code; } - template <int size> + template <int size> inline const char* UnicodeNameBySize(); - template <> + template <> inline const char* UnicodeNameBySize<1>() { return "UTF-8"; } - - template <> + + template <> inline const char* UnicodeNameBySize<2>() { return "UTF-16LE"; } - - template <> + + template <> inline const char* UnicodeNameBySize<4>() { return "UCS-4LE"; } - template <class C> + template <class C> inline const char* UnicodeName() { return UnicodeNameBySize<sizeof(C)>(); } class TDescriptor : NNonCopyable::TNonCopyable { private: - void* Descriptor_; - const char* From_; - const char* To_; + void* Descriptor_; + const char* From_; + const char* To_; public: template <class TFrom, class TTo> inline TDescriptor(TFrom from, TTo to) - : TDescriptor(CharsetName(from), CharsetName(to)) + : TDescriptor(CharsetName(from), CharsetName(to)) { } - TDescriptor(const char* from, const char* to); - - ~TDescriptor(); - - inline void* Get() const { - return Descriptor_; - } - - inline bool Invalid() const { - return Descriptor_ == (void*)(-1); + TDescriptor(const char* from, const char* to); + + ~TDescriptor(); + + inline void* Get() const { + return Descriptor_; } - inline const char* From() const noexcept { - return From_; + inline bool Invalid() const { + return Descriptor_ == (void*)(-1); } - inline const char* To() const noexcept { - return To_; + inline const char* From() const noexcept { + return From_; } + + inline const char* To() const noexcept { + return To_; + } }; template <class TFrom, class TTo> @@ -79,43 +79,43 @@ namespace NICONVPrivate { return !descriptor.Invalid(); } - size_t RecodeImpl(const TDescriptor& descriptor, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written); - void DoRecode(const TDescriptor& descriptor, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written); + size_t RecodeImpl(const TDescriptor& descriptor, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written); + void DoRecode(const TDescriptor& descriptor, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written); template <class TFrom, class TTo> inline void Recode(TFrom from, TTo to, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { TDescriptor descriptor(from, to); - DoRecode(descriptor, in, out, inSize, outSize, read, written); + DoRecode(descriptor, in, out, inSize, outSize, read, written); } - template <class TCharType> - inline void RecodeToUnicode(ECharset from, const char* in, TCharType* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { + template <class TCharType> + inline void RecodeToUnicode(ECharset from, const char* in, TCharType* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { const size_t charSize = sizeof(TCharType); Recode(from, UnicodeName<TCharType>(), in, reinterpret_cast<char*>(out), inSize, outSize * charSize, read, written); written /= charSize; } - template <class TCharType> - inline void RecodeFromUnicode(ECharset to, const TCharType* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { + template <class TCharType> + inline void RecodeFromUnicode(ECharset to, const TCharType* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { const size_t charSize = sizeof(TCharType); Recode(UnicodeName<TCharType>(), to, reinterpret_cast<const char*>(in), out, inSize * charSize, outSize, read, written); read /= charSize; } - RECODE_RESULT DoRecodeNoThrow(const TDescriptor& d, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written); - + RECODE_RESULT DoRecodeNoThrow(const TDescriptor& d, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written); + template <class TFrom, class TTo> inline RECODE_RESULT RecodeNoThrow(TFrom from, TTo to, const char* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { TDescriptor descriptor(from, to); - return DoRecodeNoThrow(descriptor, in, out, inSize, outSize, read, written); + return DoRecodeNoThrow(descriptor, in, out, inSize, outSize, read, written); } - template <class TCharType> - inline RECODE_RESULT RecodeToUnicodeNoThrow(ECharset from, const char* in, TCharType* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { + template <class TCharType> + inline RECODE_RESULT RecodeToUnicodeNoThrow(ECharset from, const char* in, TCharType* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { const size_t charSize = sizeof(TCharType); RECODE_RESULT res = RecodeNoThrow(from, UnicodeName<TCharType>(), in, reinterpret_cast<char*>(out), inSize, outSize * charSize, read, written); @@ -124,8 +124,8 @@ namespace NICONVPrivate { return res; } - template <class TCharType> - inline RECODE_RESULT RecodeFromUnicodeNoThrow(ECharset to, const TCharType* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { + template <class TCharType> + inline RECODE_RESULT RecodeFromUnicodeNoThrow(ECharset to, const TCharType* in, char* out, size_t inSize, size_t outSize, size_t& read, size_t& written) { const size_t charSize = sizeof(TCharType); RECODE_RESULT res = RecodeNoThrow(UnicodeName<TCharType>(), to, reinterpret_cast<const char*>(in), out, inSize * charSize, outSize, read, written); @@ -133,4 +133,4 @@ namespace NICONVPrivate { return res; } -} +} diff --git a/library/cpp/charset/iconv_ut.cpp b/library/cpp/charset/iconv_ut.cpp index e8c56f6d49..f79d76f7c2 100644 --- a/library/cpp/charset/iconv_ut.cpp +++ b/library/cpp/charset/iconv_ut.cpp @@ -1,7 +1,7 @@ -#include "wide.h" -#include "recyr.hh" -#include "codepage.h" - +#include "wide.h" +#include "recyr.hh" +#include "codepage.h" + #include <library/cpp/testing/unittest/registar.h> static void TestIconv(const TString& utf8, const TString& other, ECharset enc) { @@ -38,30 +38,30 @@ static void TestIconv(const TString& utf8, const TString& other, ECharset enc) { UNIT_ASSERT(temp == other); } -class TIconvTest: public TTestBase { - static void TestSurrogates(const char* str, const wchar16* wide, size_t wideSize) { - size_t sSize = strlen(str); - size_t wSize = sSize * 2; - TArrayHolder<wchar16> w(new wchar16[wSize]); - - size_t read = 0; - size_t written = 0; - NICONVPrivate::RecodeToUnicode(CODES_UTF8, str, w.Get(), sSize, wSize, read, written); - UNIT_ASSERT(read == sSize); - UNIT_ASSERT(written == wideSize); - UNIT_ASSERT(!memcmp(w.Get(), wide, wideSize)); - - TArrayHolder<char> s(new char[sSize]); - NICONVPrivate::RecodeFromUnicode(CODES_UTF8, w.Get(), s.Get(), wideSize, sSize, read, written); - UNIT_ASSERT(read == wideSize); - UNIT_ASSERT(written == sSize); - UNIT_ASSERT(!memcmp(s.Get(), str, sSize)); - } +class TIconvTest: public TTestBase { + static void TestSurrogates(const char* str, const wchar16* wide, size_t wideSize) { + size_t sSize = strlen(str); + size_t wSize = sSize * 2; + TArrayHolder<wchar16> w(new wchar16[wSize]); + + size_t read = 0; + size_t written = 0; + NICONVPrivate::RecodeToUnicode(CODES_UTF8, str, w.Get(), sSize, wSize, read, written); + UNIT_ASSERT(read == sSize); + UNIT_ASSERT(written == wideSize); + UNIT_ASSERT(!memcmp(w.Get(), wide, wideSize)); + + TArrayHolder<char> s(new char[sSize]); + NICONVPrivate::RecodeFromUnicode(CODES_UTF8, w.Get(), s.Get(), wideSize, sSize, read, written); + UNIT_ASSERT(read == wideSize); + UNIT_ASSERT(written == sSize); + UNIT_ASSERT(!memcmp(s.Get(), str, sSize)); + } private: UNIT_TEST_SUITE(TIconvTest); - UNIT_TEST(TestBig5); - UNIT_TEST(TestSurrogatePairs); + UNIT_TEST(TestBig5); + UNIT_TEST(TestSurrogatePairs); UNIT_TEST_SUITE_END(); public: @@ -75,11 +75,11 @@ public: void TestSurrogatePairs() { const char* utf8NonBMP = "\xf4\x80\x89\x84\xf4\x80\x89\x87\xf4\x80\x88\xba"; - wchar16 wNonBMPDummy[] = {0xDBC0, 0xDE44, 0xDBC0, 0xDE47, 0xDBC0, 0xDE3A}; + wchar16 wNonBMPDummy[] = {0xDBC0, 0xDE44, 0xDBC0, 0xDE47, 0xDBC0, 0xDE3A}; TestSurrogates(utf8NonBMP, wNonBMPDummy, Y_ARRAY_SIZE(wNonBMPDummy)); const char* utf8NonBMP2 = "ab\xf4\x80\x89\x87n"; - wchar16 wNonBMPDummy2[] = {'a', 'b', 0xDBC0, 0xDE47, 'n'}; + wchar16 wNonBMPDummy2[] = {'a', 'b', 0xDBC0, 0xDE47, 'n'}; TestSurrogates(utf8NonBMP2, wNonBMPDummy2, Y_ARRAY_SIZE(wNonBMPDummy2)); } }; diff --git a/library/cpp/charset/recyr.hh b/library/cpp/charset/recyr.hh index 5ec8734bcf..c5e752616e 100644 --- a/library/cpp/charset/recyr.hh +++ b/library/cpp/charset/recyr.hh @@ -14,39 +14,39 @@ /////////////////////////////////////////////////////////////////////////////////////// // input buf -> output buf // /////////////////////////////////////////////////////////////////////////////////////// -template <class TCharType> -inline RECODE_RESULT RecodeToUnicode(ECharset from, const char* in, TCharType* out, size_t inSize, size_t outSize, size_t& inRead, size_t& outWritten) { - static_assert(sizeof(TCharType) > 1, "expect wide character type"); - +template <class TCharType> +inline RECODE_RESULT RecodeToUnicode(ECharset from, const char* in, TCharType* out, size_t inSize, size_t outSize, size_t& inRead, size_t& outWritten) { + static_assert(sizeof(TCharType) > 1, "expect wide character type"); + return NCodepagePrivate::_recodeToUnicode(from, in, out, inSize, outSize, inRead, outWritten); } -template <class TCharType> -inline RECODE_RESULT RecodeFromUnicode(ECharset to, const TCharType* in, char* out, size_t inSize, size_t outSize, size_t& inRead, size_t& outWritten) { - static_assert(sizeof(TCharType) > 1, "expect wide character type"); - +template <class TCharType> +inline RECODE_RESULT RecodeFromUnicode(ECharset to, const TCharType* in, char* out, size_t inSize, size_t outSize, size_t& inRead, size_t& outWritten) { + static_assert(sizeof(TCharType) > 1, "expect wide character type"); + return NCodepagePrivate::_recodeFromUnicode(to, in, out, inSize, outSize, inRead, outWritten); } -inline RECODE_RESULT RecodeFromUnicode(ECharset to, wchar32 rune, char* out, size_t outSize, size_t& outWritten) { +inline RECODE_RESULT RecodeFromUnicode(ECharset to, wchar32 rune, char* out, size_t outSize, size_t& outWritten) { return NCodepagePrivate::_recodeFromUnicode(to, rune, out, outSize, outWritten); } -template <class TCharType> +template <class TCharType> inline RECODE_RESULT RecodeToUnicode(ECharset from, const char* in, TCharType* out, size_t inSize, size_t outSize) { size_t inRead = 0; size_t outWritten = 0; return RecodeToUnicode(from, in, out, inSize, outSize, inRead, outWritten); } -template <class TCharType> +template <class TCharType> inline RECODE_RESULT RecodeFromUnicode(ECharset to, const TCharType* in, char* out, size_t inSize, size_t outSize) { size_t inRead = 0; size_t outWritten = 0; return RecodeFromUnicode(to, in, out, inSize, outSize, inRead, outWritten); } -inline RECODE_RESULT RecodeFromUnicode(ECharset theEncoding, const wchar16* chars, size_t length, +inline RECODE_RESULT RecodeFromUnicode(ECharset theEncoding, const wchar16* chars, size_t length, char* bytes, size_t size, size_t* read = nullptr, size_t* written = nullptr) { size_t w = 0, r = 0; RECODE_RESULT rc = ::RecodeFromUnicode(theEncoding, chars, bytes, length, size, r, w); @@ -57,7 +57,7 @@ inline RECODE_RESULT RecodeFromUnicode(ECharset theEncoding, const wchar16* char return rc; } -inline RECODE_RESULT Recode(ECharset from, ECharset to, const char* in, char* out, size_t inSize, size_t outSize, size_t& inRead, size_t& outWritten) { +inline RECODE_RESULT Recode(ECharset from, ECharset to, const char* in, char* out, size_t inSize, size_t outSize, size_t& inRead, size_t& outWritten) { inRead = 0; outWritten = 0; @@ -125,7 +125,7 @@ inline bool Recode(ECharset from, ECharset to, const TStringBuf& in, TString& ou Y_ENSURE(RECODE_OK == res, "Recode failed. "); if (outWritten > outSize) ythrow yexception() << "Recode overrun the buffer: size=" - << outSize << " need=" << outWritten; + << outSize << " need=" << outWritten; out.remove(outWritten); return true; @@ -149,10 +149,10 @@ inline TString RecodeToHTMLEntities(ECharset from, const TString& in) { RECODE_RESULT res; size_t outWritten, inRead; TString out; - out.resize(in.length() * (4 + 4)); + out.resize(in.length() * (4 + 4)); res = NCodepagePrivate::_recodeToHTMLEntities(from, in.c_str(), out.begin(), in.length(), out.length(), inRead, outWritten); - if (res == RECODE_EOOUTPUT) { //input contains many 8-byte characters? - out.resize(in.length() * (4 + 8)); + if (res == RECODE_EOOUTPUT) { //input contains many 8-byte characters? + out.resize(in.length() * (4 + 8)); res = NCodepagePrivate::_recodeToHTMLEntities(from, in.c_str(), out.begin(), in.length(), out.length(), inRead, outWritten); } if (res != RECODE_OK) { diff --git a/library/cpp/charset/recyr_int.hh b/library/cpp/charset/recyr_int.hh index 353af53305..c61822037f 100644 --- a/library/cpp/charset/recyr_int.hh +++ b/library/cpp/charset/recyr_int.hh @@ -5,332 +5,332 @@ #include <util/generic/ptr.h> #include <util/generic/string.h> #include <util/system/defaults.h> - + #include "codepage.h" #include "doccodes.h" #include "iconv.h" #include "wide.h" namespace NCodepagePrivate { - inline RECODE_RESULT _recodeCopy(const char* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - in_readed = in_size; - RECODE_RESULT res = RECODE_OK; - if (in_readed > out_size) { - res = RECODE_EOOUTPUT; - in_readed = out_size; - } - if (in != out) - memcpy(out, in, in_readed); - out_writed = in_readed; - return res; + inline RECODE_RESULT _recodeCopy(const char* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + in_readed = in_size; + RECODE_RESULT res = RECODE_OK; + if (in_readed > out_size) { + res = RECODE_EOOUTPUT; + in_readed = out_size; + } + if (in != out) + memcpy(out, in, in_readed); + out_writed = in_readed; + return res; } - inline RECODE_RESULT _recodeToUTF8(ECharset From, const char* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - if (From == CODES_UTF8) - return _recodeCopy(in, out, in_size, out_size, in_readed, out_writed); - const CodePage* cp = CodePageByCharset(From); + inline RECODE_RESULT _recodeToUTF8(ECharset From, const char* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + if (From == CODES_UTF8) + return _recodeCopy(in, out, in_size, out_size, in_readed, out_writed); + const CodePage* cp = CodePageByCharset(From); - const unsigned char* in_start = (const unsigned char*)in; - const unsigned char* in_end = in_start + in_size; - const unsigned char* out_start = (unsigned char*)out; - const unsigned char* out_end = out_start + out_size; + const unsigned char* in_start = (const unsigned char*)in; + const unsigned char* in_end = in_start + in_size; + const unsigned char* out_start = (unsigned char*)out; + const unsigned char* out_end = out_start + out_size; - size_t rune_len; - RECODE_RESULT res = RECODE_OK; - while ((unsigned char*)in < in_end && res == RECODE_OK) { + size_t rune_len; + RECODE_RESULT res = RECODE_OK; + while ((unsigned char*)in < in_end && res == RECODE_OK) { res = SafeWriteUTF8Char(cp->unicode[(unsigned char)(*in++)], rune_len, (unsigned char*)out, out_end); - out += rune_len; - } - in_readed = (unsigned char*)in - in_start; - out_writed = (unsigned char*)out - out_start; - return res; + out += rune_len; + } + in_readed = (unsigned char*)in - in_start; + out_writed = (unsigned char*)out - out_start; + return res; } - inline RECODE_RESULT _recodeFromUTF8(ECharset to, const char* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - if (to == CODES_UTF8) - return _recodeCopy(in, out, in_size, out_size, in_readed, out_writed); + inline RECODE_RESULT _recodeFromUTF8(ECharset to, const char* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + if (to == CODES_UTF8) + return _recodeCopy(in, out, in_size, out_size, in_readed, out_writed); Y_ASSERT(CODES_UNKNOWN < to && to < CODES_MAX); - const Encoder* enc = &EncoderByCharset(to); + const Encoder* enc = &EncoderByCharset(to); - const unsigned char* in_start = (const unsigned char*)in; - const unsigned char* in_end = in_start + in_size; - const unsigned char* out_start = (unsigned char*)out; - const unsigned char* out_end = out_start + out_size; + const unsigned char* in_start = (const unsigned char*)in; + const unsigned char* in_end = in_start + in_size; + const unsigned char* out_start = (unsigned char*)out; + const unsigned char* out_end = out_start + out_size; - wchar32 rune; - size_t rune_len; - RECODE_RESULT res = RECODE_OK; - while ((const unsigned char*)in < in_end && (res == RECODE_OK || res == RECODE_BROKENSYMBOL)) { + wchar32 rune; + size_t rune_len; + RECODE_RESULT res = RECODE_OK; + while ((const unsigned char*)in < in_end && (res == RECODE_OK || res == RECODE_BROKENSYMBOL)) { res = SafeReadUTF8Char(rune, rune_len, (const unsigned char*)in, in_end); - if (res == RECODE_BROKENSYMBOL) - rune_len = 1; - if (res != RECODE_EOINPUT) - *out++ = enc->Tr(rune); - in += rune_len; - if (res == RECODE_OK && (const unsigned char*)in < in_end && (unsigned char*)out >= out_end) - res = RECODE_EOOUTPUT; - } - in_readed = (unsigned char*)in - in_start; - out_writed = (unsigned char*)out - out_start; - return res; + if (res == RECODE_BROKENSYMBOL) + rune_len = 1; + if (res != RECODE_EOINPUT) + *out++ = enc->Tr(rune); + in += rune_len; + if (res == RECODE_OK && (const unsigned char*)in < in_end && (unsigned char*)out >= out_end) + res = RECODE_EOOUTPUT; + } + in_readed = (unsigned char*)in - in_start; + out_writed = (unsigned char*)out - out_start; + return res; } - inline RECODE_RESULT _recodeToYandex(ECharset From, const char* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - if (From == CODES_YANDEX) - return _recodeCopy(in, out, in_size, out_size, in_readed, out_writed); - if (From == CODES_UTF8) - return _recodeFromUTF8(CODES_YANDEX, in, out, in_size, out_size, in_readed, out_writed); - in_readed = (out_size > in_size) ? in_size : out_size; - const Recoder& rcdr = NCodepagePrivate::TCodePageData::rcdr_to_yandex[From]; - rcdr.Tr(in, out, in_readed); - out_writed = in_readed; - if (out_size < in_size) - return RECODE_EOOUTPUT; - return RECODE_OK; - } - inline RECODE_RESULT _recodeFromYandex(ECharset To, const char* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - if (To == CODES_YANDEX) - return _recodeCopy(in, out, in_size, out_size, in_readed, out_writed); - if (To == CODES_UTF8) - return _recodeToUTF8(CODES_YANDEX, in, out, in_size, out_size, in_readed, out_writed); - in_readed = (out_size > in_size) ? in_size : out_size; - const Recoder& rcdr = NCodepagePrivate::TCodePageData::rcdr_from_yandex[To]; - rcdr.Tr(in, out, in_readed); - out_writed = in_readed; - if (out_size < in_size) - return RECODE_EOOUTPUT; - return RECODE_OK; - } - - template <class TCharType> - inline RECODE_RESULT _recodeUTF8ToUnicode(const char* in, TCharType* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - const unsigned char* inp = (const unsigned char*)in; - const unsigned char* in_end = inp + in_size; - TCharType* outp = out; - const TCharType* out_end = outp + out_size; - size_t rune_len; - wchar32 rune; - RECODE_RESULT res = RECODE_OK; - while ((res == RECODE_OK || res == RECODE_BROKENSYMBOL) && inp < in_end && outp < out_end) { + inline RECODE_RESULT _recodeToYandex(ECharset From, const char* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + if (From == CODES_YANDEX) + return _recodeCopy(in, out, in_size, out_size, in_readed, out_writed); + if (From == CODES_UTF8) + return _recodeFromUTF8(CODES_YANDEX, in, out, in_size, out_size, in_readed, out_writed); + in_readed = (out_size > in_size) ? in_size : out_size; + const Recoder& rcdr = NCodepagePrivate::TCodePageData::rcdr_to_yandex[From]; + rcdr.Tr(in, out, in_readed); + out_writed = in_readed; + if (out_size < in_size) + return RECODE_EOOUTPUT; + return RECODE_OK; + } + inline RECODE_RESULT _recodeFromYandex(ECharset To, const char* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + if (To == CODES_YANDEX) + return _recodeCopy(in, out, in_size, out_size, in_readed, out_writed); + if (To == CODES_UTF8) + return _recodeToUTF8(CODES_YANDEX, in, out, in_size, out_size, in_readed, out_writed); + in_readed = (out_size > in_size) ? in_size : out_size; + const Recoder& rcdr = NCodepagePrivate::TCodePageData::rcdr_from_yandex[To]; + rcdr.Tr(in, out, in_readed); + out_writed = in_readed; + if (out_size < in_size) + return RECODE_EOOUTPUT; + return RECODE_OK; + } + + template <class TCharType> + inline RECODE_RESULT _recodeUTF8ToUnicode(const char* in, TCharType* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + const unsigned char* inp = (const unsigned char*)in; + const unsigned char* in_end = inp + in_size; + TCharType* outp = out; + const TCharType* out_end = outp + out_size; + size_t rune_len; + wchar32 rune; + RECODE_RESULT res = RECODE_OK; + while ((res == RECODE_OK || res == RECODE_BROKENSYMBOL) && inp < in_end && outp < out_end) { res = SafeReadUTF8Char(rune, rune_len, inp, in_end); - if (res == RECODE_BROKENSYMBOL) - rune_len = 1; - if (res == RECODE_OK || res == RECODE_BROKENSYMBOL) { - if (!WriteSymbol(rune, outp, out_end)) { - break; - } - inp += rune_len; + if (res == RECODE_BROKENSYMBOL) + rune_len = 1; + if (res == RECODE_OK || res == RECODE_BROKENSYMBOL) { + if (!WriteSymbol(rune, outp, out_end)) { + break; + } + inp += rune_len; } } - in_readed = inp - (const unsigned char*)in; - out_writed = outp - out; - - if ((res == RECODE_OK || res == RECODE_BROKENSYMBOL) && in_readed != in_size) - return RECODE_EOOUTPUT; - - return res; - } - - template <class TCharType> - inline RECODE_RESULT _recodeSBToUnicode(ECharset From, const char* in, TCharType* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - const CodePage* cp = CodePageByCharset(From); - const unsigned char* inp = (const unsigned char*)in; - const unsigned char* in_end = inp + in_size; - TCharType* outp = out; - const TCharType* out_end = outp + out_size; - while (inp < in_end && outp < out_end) - *outp++ = static_cast<TCharType>(cp->unicode[*inp++]); - in_readed = inp - (const unsigned char*)in; - out_writed = outp - out; - if (in_readed != in_size) - return RECODE_EOOUTPUT; - return RECODE_OK; - } - - template <class TCharType> - inline RECODE_RESULT _recodeUnicodeToUTF8Impl(const TCharType* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - const TCharType* inp = in; - const TCharType* in_end = in + in_size; - unsigned char* outp = (unsigned char*)out; - const unsigned char* out_end = outp + out_size; - size_t rune_len; - wchar32 rune; - RECODE_RESULT res = RECODE_OK; - - while ((res == RECODE_OK || res == RECODE_BROKENSYMBOL) && inp != in_end) { - rune = ReadSymbolAndAdvance(inp, in_end); + in_readed = inp - (const unsigned char*)in; + out_writed = outp - out; + + if ((res == RECODE_OK || res == RECODE_BROKENSYMBOL) && in_readed != in_size) + return RECODE_EOOUTPUT; + + return res; + } + + template <class TCharType> + inline RECODE_RESULT _recodeSBToUnicode(ECharset From, const char* in, TCharType* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + const CodePage* cp = CodePageByCharset(From); + const unsigned char* inp = (const unsigned char*)in; + const unsigned char* in_end = inp + in_size; + TCharType* outp = out; + const TCharType* out_end = outp + out_size; + while (inp < in_end && outp < out_end) + *outp++ = static_cast<TCharType>(cp->unicode[*inp++]); + in_readed = inp - (const unsigned char*)in; + out_writed = outp - out; + if (in_readed != in_size) + return RECODE_EOOUTPUT; + return RECODE_OK; + } + + template <class TCharType> + inline RECODE_RESULT _recodeUnicodeToUTF8Impl(const TCharType* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + const TCharType* inp = in; + const TCharType* in_end = in + in_size; + unsigned char* outp = (unsigned char*)out; + const unsigned char* out_end = outp + out_size; + size_t rune_len; + wchar32 rune; + RECODE_RESULT res = RECODE_OK; + + while ((res == RECODE_OK || res == RECODE_BROKENSYMBOL) && inp != in_end) { + rune = ReadSymbolAndAdvance(inp, in_end); res = SafeWriteUTF8Char(rune, rune_len, outp, out_end); - if (outp >= out_end && (res == RECODE_OK || res == RECODE_BROKENSYMBOL)) - res = RECODE_EOOUTPUT; - outp += rune_len; - } - in_readed = inp - in; - out_writed = outp - (const unsigned char*)out; - return res; + if (outp >= out_end && (res == RECODE_OK || res == RECODE_BROKENSYMBOL)) + res = RECODE_EOOUTPUT; + outp += rune_len; + } + in_readed = inp - in; + out_writed = outp - (const unsigned char*)out; + return res; } - inline RECODE_RESULT _recodeUnicodeToUTF8(wchar32 rune, char* out, size_t out_size, size_t& nwritten) { + inline RECODE_RESULT _recodeUnicodeToUTF8(wchar32 rune, char* out, size_t out_size, size_t& nwritten) { return SafeWriteUTF8Char(rune, nwritten, (unsigned char*)out, out_size); - } + } - template <class TCharType, int Size = sizeof(TCharType)> - struct TCharTypeSwitch; + template <class TCharType, int Size = sizeof(TCharType)> + struct TCharTypeSwitch; - template <class TCharType> - struct TCharTypeSwitch<TCharType, 2> { + template <class TCharType> + struct TCharTypeSwitch<TCharType, 2> { using TRealCharType = wchar16; - }; + }; - template <class TCharType> - struct TCharTypeSwitch<TCharType, 4> { + template <class TCharType> + struct TCharTypeSwitch<TCharType, 4> { using TRealCharType = wchar32; - }; - - template <class TCharType> - inline RECODE_RESULT _recodeUnicodeToUTF8(const TCharType* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - static_assert(sizeof(TCharType) > 1, "expect some wide type"); + }; + template <class TCharType> + inline RECODE_RESULT _recodeUnicodeToUTF8(const TCharType* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + static_assert(sizeof(TCharType) > 1, "expect some wide type"); + using TRealCharType = typename TCharTypeSwitch<TCharType>::TRealCharType; - return _recodeUnicodeToUTF8Impl(reinterpret_cast<const TRealCharType*>(in), out, in_size, out_size, in_readed, out_writed); - } - - template <class TCharType> - inline RECODE_RESULT _recodeUnicodeToSB(ECharset To, const TCharType* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - const TCharType* inp = in; - const TCharType* in_end = in + in_size; - const char* out_begin = out; - const char* out_end = out + out_size; - - const Encoder* enc = &EncoderByCharset(To); - while (inp != in_end && out != out_end) { - *out++ = enc->Tr(ReadSymbolAndAdvance(inp, in_end)); - } - - in_readed = inp - in; - out_writed = out - out_begin; - - if (in_readed != in_size) - return RECODE_EOOUTPUT; - - return RECODE_OK; - } - - inline RECODE_RESULT _recodeUnicodeToSB(ECharset To, wchar32 rune, char* out, size_t out_size, size_t& nwritten) { - if (0 == out_size) - return RECODE_EOOUTPUT; - *out = EncoderByCharset(To).Tr(rune); - nwritten = 1; - return RECODE_OK; - } - - inline RECODE_RESULT _rune2hex(wchar32 in, char* out, size_t out_size, size_t& out_writed) { - static const char hex_digs[] = "0123456789ABCDEF"; - out_writed = 0; - RECODE_RESULT res = RECODE_OK; - for (int i = 7; i >= 0; i--) { - unsigned char h = (unsigned char)(in >> (i * 4) & 0x0F); - if (h || i == 0) { - if (out_writed + 1 >= out_size) { - res = RECODE_EOOUTPUT; - break; - } - out[out_writed++] = hex_digs[h]; + return _recodeUnicodeToUTF8Impl(reinterpret_cast<const TRealCharType*>(in), out, in_size, out_size, in_readed, out_writed); + } + + template <class TCharType> + inline RECODE_RESULT _recodeUnicodeToSB(ECharset To, const TCharType* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + const TCharType* inp = in; + const TCharType* in_end = in + in_size; + const char* out_begin = out; + const char* out_end = out + out_size; + + const Encoder* enc = &EncoderByCharset(To); + while (inp != in_end && out != out_end) { + *out++ = enc->Tr(ReadSymbolAndAdvance(inp, in_end)); + } + + in_readed = inp - in; + out_writed = out - out_begin; + + if (in_readed != in_size) + return RECODE_EOOUTPUT; + + return RECODE_OK; + } + + inline RECODE_RESULT _recodeUnicodeToSB(ECharset To, wchar32 rune, char* out, size_t out_size, size_t& nwritten) { + if (0 == out_size) + return RECODE_EOOUTPUT; + *out = EncoderByCharset(To).Tr(rune); + nwritten = 1; + return RECODE_OK; + } + + inline RECODE_RESULT _rune2hex(wchar32 in, char* out, size_t out_size, size_t& out_writed) { + static const char hex_digs[] = "0123456789ABCDEF"; + out_writed = 0; + RECODE_RESULT res = RECODE_OK; + for (int i = 7; i >= 0; i--) { + unsigned char h = (unsigned char)(in >> (i * 4) & 0x0F); + if (h || i == 0) { + if (out_writed + 1 >= out_size) { + res = RECODE_EOOUTPUT; + break; + } + out[out_writed++] = hex_digs[h]; } } - return res; + return res; } - inline RECODE_RESULT _recodeUnicodeToHTMLEntities(const wchar32* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - const wchar32* in_end = in + in_size; - const char* out_beg = out; - const wchar32* in_beg = in; - RECODE_RESULT res = RECODE_OK; - - const char* out_end = out + out_size - 1; - while (in < in_end && out < out_end) { - if (*in < 0x80 && *in != '<' && *in != '&' && *in != '>') { //ascii - *out++ = char(*in & 0x00FF); - } else { //entity - char* ent = out; - size_t ent_writed; - if (ent > out_end - 6) { - res = RECODE_EOOUTPUT; - break; - } - memcpy(ent, "&#x", 3); - ent += 3; - res = _rune2hex(*in, ent, out_end - 1 - ent, ent_writed); - if (res != RECODE_OK) - break; - ent += ent_writed; - *ent++ = ';'; - out = ent; + inline RECODE_RESULT _recodeUnicodeToHTMLEntities(const wchar32* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + const wchar32* in_end = in + in_size; + const char* out_beg = out; + const wchar32* in_beg = in; + RECODE_RESULT res = RECODE_OK; + + const char* out_end = out + out_size - 1; + while (in < in_end && out < out_end) { + if (*in < 0x80 && *in != '<' && *in != '&' && *in != '>') { //ascii + *out++ = char(*in & 0x00FF); + } else { //entity + char* ent = out; + size_t ent_writed; + if (ent > out_end - 6) { + res = RECODE_EOOUTPUT; + break; + } + memcpy(ent, "&#x", 3); + ent += 3; + res = _rune2hex(*in, ent, out_end - 1 - ent, ent_writed); + if (res != RECODE_OK) + break; + ent += ent_writed; + *ent++ = ';'; + out = ent; } - in++; + in++; } - *out++ = '\x00'; - out_writed = out - out_beg; - in_readed = in - in_beg; - return res; + *out++ = '\x00'; + out_writed = out - out_beg; + in_readed = in - in_beg; + return res; } - template <class TCharType> - inline RECODE_RESULT _recodeToUnicode(ECharset From, const char* in, TCharType* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - if (!ValidCodepage(From)) - return RECODE_ERROR; + template <class TCharType> + inline RECODE_RESULT _recodeToUnicode(ECharset From, const char* in, TCharType* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + if (!ValidCodepage(From)) + return RECODE_ERROR; - if (!NCodepagePrivate::NativeCodepage(From)) - return NICONVPrivate::RecodeToUnicodeNoThrow(From, in, out, in_size, out_size, in_readed, out_writed); + if (!NCodepagePrivate::NativeCodepage(From)) + return NICONVPrivate::RecodeToUnicodeNoThrow(From, in, out, in_size, out_size, in_readed, out_writed); - if (From == CODES_UTF8) - return _recodeUTF8ToUnicode(in, out, in_size, out_size, in_readed, out_writed); + if (From == CODES_UTF8) + return _recodeUTF8ToUnicode(in, out, in_size, out_size, in_readed, out_writed); - return _recodeSBToUnicode(From, in, out, in_size, out_size, in_readed, out_writed); - } + return _recodeSBToUnicode(From, in, out, in_size, out_size, in_readed, out_writed); + } - template <class TCharType> - inline RECODE_RESULT _recodeFromUnicode(ECharset To, const TCharType* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - if (!ValidCodepage(To)) - return RECODE_ERROR; + template <class TCharType> + inline RECODE_RESULT _recodeFromUnicode(ECharset To, const TCharType* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + if (!ValidCodepage(To)) + return RECODE_ERROR; - if (!NCodepagePrivate::NativeCodepage(To)) - return NICONVPrivate::RecodeFromUnicodeNoThrow(To, in, out, in_size, out_size, in_readed, out_writed); + if (!NCodepagePrivate::NativeCodepage(To)) + return NICONVPrivate::RecodeFromUnicodeNoThrow(To, in, out, in_size, out_size, in_readed, out_writed); - if (To == CODES_UTF8) - return NCodepagePrivate::_recodeUnicodeToUTF8(in, out, in_size, out_size, in_readed, out_writed); + if (To == CODES_UTF8) + return NCodepagePrivate::_recodeUnicodeToUTF8(in, out, in_size, out_size, in_readed, out_writed); - return NCodepagePrivate::_recodeUnicodeToSB(To, in, out, in_size, out_size, in_readed, out_writed); + return NCodepagePrivate::_recodeUnicodeToSB(To, in, out, in_size, out_size, in_readed, out_writed); } - inline RECODE_RESULT _recodeFromUnicode(ECharset To, wchar32 rune, char* out, size_t out_size, size_t& nwritten) { - if (!ValidCodepage(To)) - return RECODE_ERROR; + inline RECODE_RESULT _recodeFromUnicode(ECharset To, wchar32 rune, char* out, size_t out_size, size_t& nwritten) { + if (!ValidCodepage(To)) + return RECODE_ERROR; - if (!NCodepagePrivate::NativeCodepage(To)) { - size_t nread = 0; - return NICONVPrivate::RecodeFromUnicodeNoThrow(To, &rune, out, 1, out_size, nread, nwritten); - } + if (!NCodepagePrivate::NativeCodepage(To)) { + size_t nread = 0; + return NICONVPrivate::RecodeFromUnicodeNoThrow(To, &rune, out, 1, out_size, nread, nwritten); + } - if (To == CODES_UTF8) - return NCodepagePrivate::_recodeUnicodeToUTF8(rune, out, out_size, nwritten); + if (To == CODES_UTF8) + return NCodepagePrivate::_recodeUnicodeToUTF8(rune, out, out_size, nwritten); - return NCodepagePrivate::_recodeUnicodeToSB(To, rune, out, out_size, nwritten); - } + return NCodepagePrivate::_recodeUnicodeToSB(To, rune, out, out_size, nwritten); + } - inline RECODE_RESULT _recodeToHTMLEntities(ECharset From, const char* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { - TArrayHolder<wchar32> bufHolder(new wchar32[in_size]); - wchar32* buf = bufHolder.Get(); - size_t unicode_size; - RECODE_RESULT res1, res2; + inline RECODE_RESULT _recodeToHTMLEntities(ECharset From, const char* in, char* out, size_t in_size, size_t out_size, size_t& in_readed, size_t& out_writed) { + TArrayHolder<wchar32> bufHolder(new wchar32[in_size]); + wchar32* buf = bufHolder.Get(); + size_t unicode_size; + RECODE_RESULT res1, res2; - //first pass - to unicode - res1 = _recodeToUnicode(From, in, buf, in_size, in_size, in_readed, unicode_size); + //first pass - to unicode + res1 = _recodeToUnicode(From, in, buf, in_size, in_size, in_readed, unicode_size); - //second pass - to entities - res2 = _recodeUnicodeToHTMLEntities(buf, out, in_size, out_size, in_readed, out_writed); + //second pass - to entities + res2 = _recodeUnicodeToHTMLEntities(buf, out, in_size, out_size, in_readed, out_writed); - return (res2 != RECODE_OK) ? res2 : res1; - } + return (res2 != RECODE_OK) ? res2 : res1; + } -} +} diff --git a/library/cpp/charset/wide.h b/library/cpp/charset/wide.h index 32d30e849e..b7a391f0a5 100644 --- a/library/cpp/charset/wide.h +++ b/library/cpp/charset/wide.h @@ -1,15 +1,15 @@ #pragma once #include "codepage.h" -#include "iconv.h" - +#include "iconv.h" + #include <util/charset/recode_result.h> #include <util/charset/unidata.h> #include <util/charset/utf8.h> #include <util/charset/wide.h> #include <util/generic/string.h> #include <util/generic/algorithm.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <util/memory/tempbuf.h> #include <util/system/yassert.h> @@ -19,7 +19,7 @@ template <typename TCharType> inline size_t WideToChar(const TCharType* text, size_t len, char* dest, ECharset enc) { Y_ASSERT(SingleByteCodepage(enc)); - + const char* start = dest; const Encoder* const encoder = &EncoderByCharset(enc); @@ -114,7 +114,7 @@ namespace NDetail { return RecodeMultiByteChar(src, dst, encoding); } - } + } template <typename TCharFrom> struct TRecodeTraits; @@ -124,8 +124,8 @@ namespace NDetail { using TCharTo = wchar16; using TStringBufTo = TWtringBuf; using TStringTo = TUtf16String; - enum { ReserveSize = 4 }; // How many TCharFrom characters we should reserve for one TCharTo character in worst case - // Here an unicode character can be converted up to 4 bytes of UTF8 + enum { ReserveSize = 4 }; // How many TCharFrom characters we should reserve for one TCharTo character in worst case + // Here an unicode character can be converted up to 4 bytes of UTF8 }; template <> @@ -133,7 +133,7 @@ namespace NDetail { using TCharTo = char; using TStringBufTo = TStringBuf; using TStringTo = TString; - enum { ReserveSize = 2 }; // possible surrogate pairs ? + enum { ReserveSize = 2 }; // possible surrogate pairs ? }; // Operations with destination buffer where recoded string will be written @@ -203,7 +203,7 @@ namespace NDetail { Recode<TCharFrom>(src, res, encoding); return res; } -} +} // Write result into @dst. Return string-buffer pointing to re-coded content of @dst. @@ -291,7 +291,7 @@ inline TString WideToChar(const TWtringBuf w, ECharset enc) { inline TUtf16String CharToWide(const TStringBuf s, ECharset enc) { return CharToWide<false>(s.data(), s.size(), enc); } - + template <bool robust> inline TUtf16String CharToWide(const TStringBuf s, ECharset enc) { return CharToWide<robust>(s.data(), s.size(), enc); diff --git a/library/cpp/charset/wide_ut.cpp b/library/cpp/charset/wide_ut.cpp index 78947d51ba..fc727fb1b4 100644 --- a/library/cpp/charset/wide_ut.cpp +++ b/library/cpp/charset/wide_ut.cpp @@ -1,14 +1,14 @@ -#include "wide.h" -#include "codepage.h" +#include "wide.h" +#include "codepage.h" #include "recyr.hh" - + #include <library/cpp/testing/unittest/registar.h> - + #include <util/charset/utf8.h> -#include <util/digest/numeric.h> +#include <util/digest/numeric.h> #include <util/generic/hash_set.h> -#include <algorithm> +#include <algorithm> namespace { //! three UTF8 encoded russian letters (A, B, V) @@ -21,7 +21,7 @@ namespace { 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, - 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, 0x00}; + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, 0x00}; const char utf8CyrillicAlphabet[] = "\xd0\x90\xd0\x91\xd0\x92\xd0\x93\xd0\x94\xd0\x95\xd0\x96\xd0\x97" "\xd0\x98\xd0\x99\xd0\x9a\xd0\x9b\xd0\x9c\xd0\x9d\xd0\x9e\xd0\x9f" @@ -34,7 +34,7 @@ namespace { TString CreateYandexText() { const int len = 256; - char text[len] = {0}; + char text[len] = {0}; for (int i = 0; i < len; ++i) { text[i] = static_cast<char>(i); } @@ -61,7 +61,7 @@ namespace { for (int i = 0; i < len; ++i) { if (i <= 0x7F) { // ASCII characters without 0x7 and 0x1B text[i] = static_cast<wchar16>(i); - } else if (i >= 0xC0 && i <= 0xFF) { // russian characters (without YO and yo) + } else if (i >= 0xC0 && i <= 0xFF) { // russian characters (without YO and yo) text[i] = static_cast<wchar16>(i + 0x0350); // 0x0410 - 0x044F } } @@ -94,27 +94,27 @@ namespace { '\xd0', '\xb7', '\xd0', '\xb8', '\xd0', '\xb9', '\xd0', '\xba', '\xd0', '\xbb', '\xd0', '\xbc', '\xd0', '\xbd', '\xd0', '\xbe', '\xd0', '\xbf', '\xd1', '\x80', '\xd1', '\x81', '\xd1', '\x82', '\xd1', '\x83', '\xd1', '\x84', '\xd1', '\x85', '\xd1', '\x86', '\xd1', '\x87', '\xd1', '\x88', '\xd1', '\x89', '\xd1', '\x8a', '\xd1', '\x8b', '\xd1', '\x8c', '\xd1', '\x8d', '\xd1', '\x8e', - '\xd1', '\x8f'}; + '\xd1', '\x8f'}; return TString(text, Y_ARRAY_SIZE(text)); } //! use this function to dump UTF8 text into a file in case of any changes - // void DumpUTF8Text() { + // void DumpUTF8Text() { // TString s = WideToUTF8(UnicodeText); - // std::ofstream f("utf8.txt"); - // f << std::hex; - // for (int i = 0; i < (int)s.size(); ++i) { - // f << "0x" << std::setw(2) << std::setfill('0') << (int)(ui8)s[i] << ", "; - // if ((i + 1) % 16 == 0) - // f << std::endl; - // } - // } + // std::ofstream f("utf8.txt"); + // f << std::hex; + // for (int i = 0; i < (int)s.size(); ++i) { + // f << "0x" << std::setw(2) << std::setfill('0') << (int)(ui8)s[i] << ", "; + // if ((i + 1) % 16 == 0) + // f << std::endl; + // } + // } } //! this unit tests ensure validity of Yandex-Unicode and UTF8-Unicode conversions //! @note only those conversions are verified because they are used in index -class TConversionTest: public TTestBase { +class TConversionTest: public TTestBase { private: //! @note every of the text can have zeros in the middle const TString YandexText; @@ -123,13 +123,13 @@ private: private: UNIT_TEST_SUITE(TConversionTest); - UNIT_TEST(TestCharToWide); - UNIT_TEST(TestWideToChar); + UNIT_TEST(TestCharToWide); + UNIT_TEST(TestWideToChar); UNIT_TEST(TestYandexEncoding); - UNIT_TEST(TestRecodeIntoString); - UNIT_TEST(TestRecodeAppend); - UNIT_TEST(TestRecode); - UNIT_TEST(TestUnicodeLimit); + UNIT_TEST(TestRecodeIntoString); + UNIT_TEST(TestRecodeAppend); + UNIT_TEST(TestRecode); + UNIT_TEST(TestUnicodeLimit); UNIT_TEST_SUITE_END(); public: @@ -152,23 +152,23 @@ public: UNIT_TEST_SUITE_REGISTRATION(TConversionTest); // test conversions (char -> wchar32), (wchar32 -> char) and (wchar32 -> wchar16) -#define TEST_WCHAR32(sbuf, wbuf, enc) \ - do { \ - /* convert char to wchar32 */ \ - TTempBuf tmpbuf1(sbuf.length() * sizeof(wchar32)); \ +#define TEST_WCHAR32(sbuf, wbuf, enc) \ + do { \ + /* convert char to wchar32 */ \ + TTempBuf tmpbuf1(sbuf.length() * sizeof(wchar32)); \ const TBasicStringBuf<wchar32> s4buf = NDetail::NBaseOps::Recode<char>(sbuf, reinterpret_cast<wchar32*>(tmpbuf1.Data()), enc); \ - \ - /* convert wchar32 to char */ \ - TTempBuf tmpbuf2(s4buf.length() * 4); \ - const TStringBuf s1buf = NDetail::NBaseOps::Recode(s4buf, tmpbuf2.Data(), enc); \ - \ - /* convert wchar32 to wchar16 */ \ - const TUtf16String wstr2 = UTF32ToWide(s4buf.data(), s4buf.length()); \ - \ - /* test conversions */ \ - UNIT_ASSERT_VALUES_EQUAL(sbuf, s1buf); \ - UNIT_ASSERT_VALUES_EQUAL(wbuf, wstr2); \ - } while (false) + \ + /* convert wchar32 to char */ \ + TTempBuf tmpbuf2(s4buf.length() * 4); \ + const TStringBuf s1buf = NDetail::NBaseOps::Recode(s4buf, tmpbuf2.Data(), enc); \ + \ + /* convert wchar32 to wchar16 */ \ + const TUtf16String wstr2 = UTF32ToWide(s4buf.data(), s4buf.length()); \ + \ + /* test conversions */ \ + UNIT_ASSERT_VALUES_EQUAL(sbuf, s1buf); \ + UNIT_ASSERT_VALUES_EQUAL(wbuf, wstr2); \ + } while (false) void TConversionTest::TestCharToWide() { TUtf16String w = CharToWide(YandexText, CODES_YANDEX); @@ -210,7 +210,7 @@ void TConversionTest::TestYandexEncoding() { UNIT_ASSERT(w == wideCyrillicAlphabet); const char* utf8NonBMP2 = "ab\xf4\x80\x89\x87n"; - wchar16 wNonBMPDummy2[] = {'a', 'b', 0xDBC0, 0xDE47, 'n'}; + wchar16 wNonBMPDummy2[] = {'a', 'b', 0xDBC0, 0xDE47, 'n'}; TestSurrogates(utf8NonBMP2, wNonBMPDummy2, Y_ARRAY_SIZE(wNonBMPDummy2), CODES_UTF8); { @@ -232,7 +232,7 @@ void TConversionTest::TestRecodeIntoString() { TString sYandex(UnicodeText.size() * 4, 'x'); const char* sdata = sYandex.data(); TStringBuf sres = NDetail::Recode<wchar16>(UnicodeText, sYandex, CODES_YANDEX); - UNIT_ASSERT(sYandex == YandexText); // same content + UNIT_ASSERT(sYandex == YandexText); // same content UNIT_ASSERT(sYandex.data() == sdata); // reserved buffer reused UNIT_ASSERT(sYandex.data() == sres.data()); // same buffer UNIT_ASSERT(sYandex.size() == sres.size()); // same size @@ -242,7 +242,7 @@ void TConversionTest::TestRecodeIntoString() { sUnicode.reserve(YandexText.size() * 4); const wchar16* wdata = sUnicode.data(); TWtringBuf wres = NDetail::Recode<char>(YandexText, sUnicode, CODES_YANDEX); - UNIT_ASSERT(sUnicode == UnicodeText); // same content + UNIT_ASSERT(sUnicode == UnicodeText); // same content UNIT_ASSERT(sUnicode.data() == wdata); // reserved buffer reused UNIT_ASSERT(sUnicode.data() == wres.data()); // same buffer UNIT_ASSERT(sUnicode.size() == wres.size()); // same size @@ -250,8 +250,8 @@ void TConversionTest::TestRecodeIntoString() { TString sUtf8 = " "; size_t scap = sUtf8.capacity(); sres = NDetail::Recode<wchar16>(UnicodeText, sUtf8, CODES_UTF8); - UNIT_ASSERT(sUtf8 == UTF8Text); // same content - UNIT_ASSERT(sUtf8.capacity() > scap); // increased buffer capacity (supplied was too small) + UNIT_ASSERT(sUtf8 == UTF8Text); // same content + UNIT_ASSERT(sUtf8.capacity() > scap); // increased buffer capacity (supplied was too small) UNIT_ASSERT(sUtf8.data() == sres.data()); // same buffer UNIT_ASSERT(sUtf8.size() == sres.size()); // same size TEST_WCHAR32(sUtf8, UnicodeText, CODES_UTF8); @@ -260,7 +260,7 @@ void TConversionTest::TestRecodeIntoString() { wdata = sUnicode.data(); TUtf16String copy = sUnicode; // increase ref-counter wres = NDetail::Recode<char>(UTF8Text, sUnicode, CODES_UTF8); - UNIT_ASSERT(sUnicode == UnicodeText); // same content + UNIT_ASSERT(sUnicode == UnicodeText); // same content #ifndef TSTRING_IS_STD_STRING UNIT_ASSERT(sUnicode.data() != wdata); // re-allocated (shared buffer supplied) UNIT_ASSERT(sUnicode.data() == wres.data()); // same buffer diff --git a/library/cpp/charset/ya.make b/library/cpp/charset/ya.make index 7565566bf0..8906c507f0 100644 --- a/library/cpp/charset/ya.make +++ b/library/cpp/charset/ya.make @@ -1,10 +1,10 @@ -LIBRARY() +LIBRARY() OWNER(alzobnin) SRCS( - generated/cp_data.cpp - generated/encrec_data.cpp + generated/cp_data.cpp + generated/encrec_data.cpp codepage.cpp cp_encrec.cpp doccodes.cpp diff --git a/library/cpp/codecs/codecs.cpp b/library/cpp/codecs/codecs.cpp index b17a3156d2..bc60d10cf3 100644 --- a/library/cpp/codecs/codecs.cpp +++ b/library/cpp/codecs/codecs.cpp @@ -4,187 +4,187 @@ #include <util/stream/mem.h> namespace NCodecs { - void ICodec::Store(IOutputStream* out, TCodecPtr p) { - if (!p.Get()) { - ::Save(out, (ui16)0); - return; - } - - Y_ENSURE_EX(p->AlreadyTrained(), TCodecException() << "untrained codec " << p->GetName()); - const TString& n = p->GetName(); + void ICodec::Store(IOutputStream* out, TCodecPtr p) { + if (!p.Get()) { + ::Save(out, (ui16)0); + return; + } + + Y_ENSURE_EX(p->AlreadyTrained(), TCodecException() << "untrained codec " << p->GetName()); + const TString& n = p->GetName(); Y_VERIFY(n.size() <= Max<ui16>()); ::Save(out, (ui16)n.size()); out->Write(n.data(), n.size()); - p->Save(out); + p->Save(out); } - TCodecPtr ICodec::Restore(IInputStream* in) { - ui16 l = 0; - ::Load(in, l); + TCodecPtr ICodec::Restore(IInputStream* in) { + ui16 l = 0; + ::Load(in, l); - if (!l) { - return nullptr; - } + if (!l) { + return nullptr; + } - TString n; - n.resize(l); + TString n; + n.resize(l); - Y_ENSURE_EX(in->Load(n.begin(), l) == l, TCodecException()); + Y_ENSURE_EX(in->Load(n.begin(), l) == l, TCodecException()); - TCodecPtr p = ICodec::GetInstance(n); - p->Load(in); - p->Trained = true; - return p; - } + TCodecPtr p = ICodec::GetInstance(n); + p->Load(in); + p->Trained = true; + return p; + } - TCodecPtr ICodec::RestoreFromString(TStringBuf s) { + TCodecPtr ICodec::RestoreFromString(TStringBuf s) { TMemoryInput minp{s.data(), s.size()}; - return Restore(&minp); - } + return Restore(&minp); + } - TString ICodec::GetNameSafe(TCodecPtr p) { - return !p ? TString("none") : p->GetName(); - } + TString ICodec::GetNameSafe(TCodecPtr p) { + return !p ? TString("none") : p->GetName(); + } - ui8 TPipelineCodec::Encode(TStringBuf in, TBuffer& out) const { + ui8 TPipelineCodec::Encode(TStringBuf in, TBuffer& out) const { size_t res = Traits().ApproximateSizeOnEncode(in.size()); - out.Reserve(res); - out.Clear(); + out.Reserve(res); + out.Clear(); - if (Pipeline.empty()) { + if (Pipeline.empty()) { out.Append(in.data(), in.size()); - return 0; - } else if (Pipeline.size() == 1) { - return Pipeline.front()->Encode(in, out); - } + return 0; + } else if (Pipeline.size() == 1) { + return Pipeline.front()->Encode(in, out); + } - ui8 freelastbits = 0; + ui8 freelastbits = 0; - auto buffer = TBufferTlsCache::TlsInstance().Item(); - TBuffer& tmp = buffer.Get(); - tmp.Reserve(res); + auto buffer = TBufferTlsCache::TlsInstance().Item(); + TBuffer& tmp = buffer.Get(); + tmp.Reserve(res); - for (auto it = Pipeline.begin(); it != Pipeline.end(); ++it) { - if (it != Pipeline.begin()) { - tmp.Clear(); - tmp.Swap(out); + for (auto it = Pipeline.begin(); it != Pipeline.end(); ++it) { + if (it != Pipeline.begin()) { + tmp.Clear(); + tmp.Swap(out); in = TStringBuf{tmp.data(), tmp.size()}; - } - freelastbits = (*it)->Encode(in, out); - } + } + freelastbits = (*it)->Encode(in, out); + } - return freelastbits; + return freelastbits; } - void TPipelineCodec::Decode(TStringBuf in, TBuffer& out) const { + void TPipelineCodec::Decode(TStringBuf in, TBuffer& out) const { size_t res = Traits().ApproximateSizeOnDecode(in.size()); - out.Reserve(res); - out.Clear(); + out.Reserve(res); + out.Clear(); - if (Pipeline.empty()) { + if (Pipeline.empty()) { out.Append(in.data(), in.size()); - return; - } else if (Pipeline.size() == 1) { - Pipeline.front()->Decode(in, out); - return; - } + return; + } else if (Pipeline.size() == 1) { + Pipeline.front()->Decode(in, out); + return; + } - auto buffer = TBufferTlsCache::TlsInstance().Item(); + auto buffer = TBufferTlsCache::TlsInstance().Item(); - TBuffer& tmp = buffer.Get(); - tmp.Reserve(res); + TBuffer& tmp = buffer.Get(); + tmp.Reserve(res); - for (TPipeline::const_reverse_iterator it = Pipeline.rbegin(); it != Pipeline.rend(); ++it) { - if (it != Pipeline.rbegin()) { - tmp.Clear(); - tmp.Swap(out); + for (TPipeline::const_reverse_iterator it = Pipeline.rbegin(); it != Pipeline.rend(); ++it) { + if (it != Pipeline.rbegin()) { + tmp.Clear(); + tmp.Swap(out); in = TStringBuf{tmp.data(), tmp.size()}; - } - (*it)->Decode(in, out); + } + (*it)->Decode(in, out); } } - void TPipelineCodec::Save(IOutputStream* out) const { - for (const auto& it : Pipeline) - it->Save(out); + void TPipelineCodec::Save(IOutputStream* out) const { + for (const auto& it : Pipeline) + it->Save(out); } - void TPipelineCodec::Load(IInputStream* in) { - for (const auto& it : Pipeline) { - it->Load(in); - it->SetTrained(true); - } + void TPipelineCodec::Load(IInputStream* in) { + for (const auto& it : Pipeline) { + it->Load(in); + it->SetTrained(true); + } } - void TPipelineCodec::SetTrained(bool t) { - for (const auto& it : Pipeline) { - it->SetTrained(t); - } + void TPipelineCodec::SetTrained(bool t) { + for (const auto& it : Pipeline) { + it->SetTrained(t); + } } - TPipelineCodec& TPipelineCodec::AddCodec(TCodecPtr codec) { - if (!codec) - return *this; - - TCodecTraits tr = codec->Traits(); - - if (!MyName) { - MyTraits.AssumesStructuredInput = tr.AssumesStructuredInput; - MyTraits.SizeOfInputElement = tr.SizeOfInputElement; - } else { - MyName.append(':'); - } - - MyName.append(codec->GetName()); - MyTraits.PreservesPrefixGrouping &= tr.PreservesPrefixGrouping; - MyTraits.PaddingBit = tr.PaddingBit; - MyTraits.NeedsTraining |= tr.NeedsTraining; - MyTraits.Irreversible |= tr.Irreversible; - MyTraits.SizeOnEncodeAddition = MyTraits.SizeOnEncodeAddition * tr.SizeOnEncodeMultiplier + tr.SizeOnEncodeAddition; - MyTraits.SizeOnEncodeMultiplier *= tr.SizeOnEncodeMultiplier; - MyTraits.SizeOnDecodeMultiplier *= tr.SizeOnDecodeMultiplier; - MyTraits.RecommendedSampleSize = Max(MyTraits.RecommendedSampleSize, tr.RecommendedSampleSize); - - Pipeline.push_back(codec); - return *this; + TPipelineCodec& TPipelineCodec::AddCodec(TCodecPtr codec) { + if (!codec) + return *this; + + TCodecTraits tr = codec->Traits(); + + if (!MyName) { + MyTraits.AssumesStructuredInput = tr.AssumesStructuredInput; + MyTraits.SizeOfInputElement = tr.SizeOfInputElement; + } else { + MyName.append(':'); + } + + MyName.append(codec->GetName()); + MyTraits.PreservesPrefixGrouping &= tr.PreservesPrefixGrouping; + MyTraits.PaddingBit = tr.PaddingBit; + MyTraits.NeedsTraining |= tr.NeedsTraining; + MyTraits.Irreversible |= tr.Irreversible; + MyTraits.SizeOnEncodeAddition = MyTraits.SizeOnEncodeAddition * tr.SizeOnEncodeMultiplier + tr.SizeOnEncodeAddition; + MyTraits.SizeOnEncodeMultiplier *= tr.SizeOnEncodeMultiplier; + MyTraits.SizeOnDecodeMultiplier *= tr.SizeOnDecodeMultiplier; + MyTraits.RecommendedSampleSize = Max(MyTraits.RecommendedSampleSize, tr.RecommendedSampleSize); + + Pipeline.push_back(codec); + return *this; } - void TPipelineCodec::DoLearnX(ISequenceReader& in, double sampleSizeMult) { - if (!Traits().NeedsTraining) { - return; - } + void TPipelineCodec::DoLearnX(ISequenceReader& in, double sampleSizeMult) { + if (!Traits().NeedsTraining) { + return; + } - if (Pipeline.size() == 1) { - Pipeline.back()->Learn(in); - return; - } + if (Pipeline.size() == 1) { + Pipeline.back()->Learn(in); + return; + } - TVector<TBuffer> trainingInput; + TVector<TBuffer> trainingInput; - TStringBuf r; - while (in.NextRegion(r)) { + TStringBuf r; + while (in.NextRegion(r)) { trainingInput.emplace_back(r.data(), r.size()); } - - TBuffer buff; - for (const auto& it : Pipeline) { - it->LearnX(trainingInput.begin(), trainingInput.end(), sampleSizeMult); - - for (auto& bit : trainingInput) { - buff.Clear(); + + TBuffer buff; + for (const auto& it : Pipeline) { + it->LearnX(trainingInput.begin(), trainingInput.end(), sampleSizeMult); + + for (auto& bit : trainingInput) { + buff.Clear(); it->Encode(TStringBuf{bit.data(), bit.size()}, buff); - buff.Swap(bit); - } - } + buff.Swap(bit); + } + } } - bool TPipelineCodec::AlreadyTrained() const { - for (const auto& it : Pipeline) { - if (!it->AlreadyTrained()) - return false; - } - - return true; + bool TPipelineCodec::AlreadyTrained() const { + for (const auto& it : Pipeline) { + if (!it->AlreadyTrained()) + return false; + } + + return true; } } diff --git a/library/cpp/codecs/codecs.h b/library/cpp/codecs/codecs.h index cc5e72b285..08ea9beb44 100644 --- a/library/cpp/codecs/codecs.h +++ b/library/cpp/codecs/codecs.h @@ -16,244 +16,244 @@ #include <util/ysaveload.h> namespace NCodecs { - class TCodecException: public TWithBackTrace<yexception> {}; + class TCodecException: public TWithBackTrace<yexception> {}; - class ICodec; + class ICodec; - using TCodecPtr = TIntrusivePtr<ICodec>; - using TCodecConstPtr = TIntrusiveConstPtr<ICodec>; + using TCodecPtr = TIntrusivePtr<ICodec>; + using TCodecConstPtr = TIntrusiveConstPtr<ICodec>; - struct TCodecTraits { - ui32 RecommendedSampleSize = 0; - ui16 SizeOfInputElement = 1; - ui8 SizeOnEncodeMultiplier = 1; - ui8 SizeOnEncodeAddition = 0; - ui8 SizeOnDecodeMultiplier = 1; + struct TCodecTraits { + ui32 RecommendedSampleSize = 0; + ui16 SizeOfInputElement = 1; + ui8 SizeOnEncodeMultiplier = 1; + ui8 SizeOnEncodeAddition = 0; + ui8 SizeOnDecodeMultiplier = 1; - bool NeedsTraining = false; - bool PreservesPrefixGrouping = false; - bool Irreversible = false; - bool PaddingBit = 0; - bool AssumesStructuredInput = false; + bool NeedsTraining = false; + bool PreservesPrefixGrouping = false; + bool Irreversible = false; + bool PaddingBit = 0; + bool AssumesStructuredInput = false; - size_t ApproximateSizeOnEncode(size_t sz) const { - return sz * SizeOnEncodeMultiplier + SizeOnEncodeAddition; - } + size_t ApproximateSizeOnEncode(size_t sz) const { + return sz * SizeOnEncodeMultiplier + SizeOnEncodeAddition; + } - size_t ApproximateSizeOnDecode(size_t sz) const { - return sz * SizeOnDecodeMultiplier; - } - }; + size_t ApproximateSizeOnDecode(size_t sz) const { + return sz * SizeOnDecodeMultiplier; + } + }; - class ICodec: public TAtomicRefCount<ICodec> { - protected: - bool Trained = false; - TCodecTraits MyTraits; + class ICodec: public TAtomicRefCount<ICodec> { + protected: + bool Trained = false; + TCodecTraits MyTraits; - public: - TCodecTraits Traits() const { - return MyTraits; - } + public: + TCodecTraits Traits() const { + return MyTraits; + } - // the name of the codec (or its variant) to be used in the codec registry - virtual TString GetName() const = 0; + // the name of the codec (or its variant) to be used in the codec registry + virtual TString GetName() const = 0; - virtual ui8 /*free bits in last byte*/ Encode(TStringBuf, TBuffer&) const = 0; + virtual ui8 /*free bits in last byte*/ Encode(TStringBuf, TBuffer&) const = 0; virtual ui8 Encode(const TBuffer& input, TBuffer& output) const { return Encode(TStringBuf(input.Data(), input.Data() + input.Size()), output); } - virtual void Decode(TStringBuf, TBuffer&) const = 0; + virtual void Decode(TStringBuf, TBuffer&) const = 0; virtual void Decode(const TBuffer& input, TBuffer& output) const { Decode(TStringBuf(input.Data(), input.Data() + input.Size()), output); } - virtual ~ICodec() = default; + virtual ~ICodec() = default; - virtual bool AlreadyTrained() const { - return !Traits().NeedsTraining || Trained; - } - virtual void SetTrained(bool t) { - Trained = t; - } + virtual bool AlreadyTrained() const { + return !Traits().NeedsTraining || Trained; + } + virtual void SetTrained(bool t) { + Trained = t; + } bool TryToLearn(ISequenceReader& r) { Trained = DoTryToLearn(r); return Trained; } - void Learn(ISequenceReader& r) { - LearnX(r, 1); - } + void Learn(ISequenceReader& r) { + LearnX(r, 1); + } - template <class TIter> - void Learn(TIter beg, TIter end) { - Learn(beg, end, IterToStringBuf<TIter>); - } + template <class TIter> + void Learn(TIter beg, TIter end) { + Learn(beg, end, IterToStringBuf<TIter>); + } - template <class TIter, class TGetter> - void Learn(TIter beg, TIter end, TGetter getter) { - auto sample = GetSample(beg, end, Traits().RecommendedSampleSize, getter); - TSimpleSequenceReader<TBuffer> reader{sample}; - Learn(reader); - } + template <class TIter, class TGetter> + void Learn(TIter beg, TIter end, TGetter getter) { + auto sample = GetSample(beg, end, Traits().RecommendedSampleSize, getter); + TSimpleSequenceReader<TBuffer> reader{sample}; + Learn(reader); + } - static TCodecPtr GetInstance(TStringBuf name); + static TCodecPtr GetInstance(TStringBuf name); - static TVector<TString> GetCodecsList(); + static TVector<TString> GetCodecsList(); - static TString GetNameSafe(TCodecPtr p); + static TString GetNameSafe(TCodecPtr p); - static void Store(IOutputStream* out, TCodecPtr p); - static TCodecPtr Restore(IInputStream* in); - static TCodecPtr RestoreFromString(TStringBuf); + static void Store(IOutputStream* out, TCodecPtr p); + static TCodecPtr Restore(IInputStream* in); + static TCodecPtr RestoreFromString(TStringBuf); - protected: - virtual void DoLearn(ISequenceReader&) = 0; + protected: + virtual void DoLearn(ISequenceReader&) = 0; virtual bool DoTryToLearn(ISequenceReader& r) { DoLearn(r); return true; } - // so the pipeline codec will know to adjust the sample for the subcodecs - virtual void DoLearnX(ISequenceReader& r, double /*sampleSizeMultiplier*/) { - DoLearn(r); - } - - virtual void Save(IOutputStream*) const { - } - virtual void Load(IInputStream*) { - } - friend class TPipelineCodec; - - public: - // so the pipeline codec will know to adjust the sample for the subcodecs - void LearnX(ISequenceReader& r, double sampleSizeMult) { - DoLearnX(r, sampleSizeMult); - Trained = true; - } - - template <class TIter> - void LearnX(TIter beg, TIter end, double sampleSizeMult) { - auto sample = GetSample(beg, end, Traits().RecommendedSampleSize * sampleSizeMult); - TSimpleSequenceReader<TBuffer> reader{sample}; - LearnX(reader, sampleSizeMult); - } - }; - - class TBasicTrivialCodec: public ICodec { - public: - ui8 Encode(TStringBuf in, TBuffer& out) const override { + // so the pipeline codec will know to adjust the sample for the subcodecs + virtual void DoLearnX(ISequenceReader& r, double /*sampleSizeMultiplier*/) { + DoLearn(r); + } + + virtual void Save(IOutputStream*) const { + } + virtual void Load(IInputStream*) { + } + friend class TPipelineCodec; + + public: + // so the pipeline codec will know to adjust the sample for the subcodecs + void LearnX(ISequenceReader& r, double sampleSizeMult) { + DoLearnX(r, sampleSizeMult); + Trained = true; + } + + template <class TIter> + void LearnX(TIter beg, TIter end, double sampleSizeMult) { + auto sample = GetSample(beg, end, Traits().RecommendedSampleSize * sampleSizeMult); + TSimpleSequenceReader<TBuffer> reader{sample}; + LearnX(reader, sampleSizeMult); + } + }; + + class TBasicTrivialCodec: public ICodec { + public: + ui8 Encode(TStringBuf in, TBuffer& out) const override { out.Assign(in.data(), in.size()); - return 0; - } + return 0; + } - void Decode(TStringBuf in, TBuffer& out) const override { - Encode(in, out); - } + void Decode(TStringBuf in, TBuffer& out) const override { + Encode(in, out); + } - protected: - void DoLearn(ISequenceReader&) override { - } - }; + protected: + void DoLearn(ISequenceReader&) override { + } + }; - class TTrivialCodec: public TBasicTrivialCodec { - public: - TTrivialCodec() { - MyTraits.PreservesPrefixGrouping = true; - } + class TTrivialCodec: public TBasicTrivialCodec { + public: + TTrivialCodec() { + MyTraits.PreservesPrefixGrouping = true; + } - static TStringBuf MyName() { - return "trivial"; - } + static TStringBuf MyName() { + return "trivial"; + } - TString GetName() const override { + TString GetName() const override { return ToString(MyName()); - } - }; + } + }; - class TTrivialTrainableCodec: public TBasicTrivialCodec { - public: - TTrivialTrainableCodec() { - MyTraits.PreservesPrefixGrouping = true; - MyTraits.NeedsTraining = true; - } + class TTrivialTrainableCodec: public TBasicTrivialCodec { + public: + TTrivialTrainableCodec() { + MyTraits.PreservesPrefixGrouping = true; + MyTraits.NeedsTraining = true; + } - static TStringBuf MyName() { - return "trivial-trainable"; - } + static TStringBuf MyName() { + return "trivial-trainable"; + } - TString GetName() const override { + TString GetName() const override { return ToString(MyName()); - } - }; - - class TNullCodec: public ICodec { - public: - TNullCodec() { - MyTraits.Irreversible = true; - MyTraits.SizeOnDecodeMultiplier = 0; - MyTraits.SizeOnEncodeMultiplier = 0; - } - - TString GetName() const override { - return "null"; - } - - ui8 Encode(TStringBuf, TBuffer& out) const override { - out.Clear(); - return 0; - } - - void Decode(TStringBuf, TBuffer& out) const override { - out.Clear(); - } - - protected: - void DoLearn(ISequenceReader&) override { - } - }; - - class TPipelineCodec: public ICodec { - typedef TVector<TCodecPtr> TPipeline; - - TPipeline Pipeline; - TString MyName; - - public: - explicit TPipelineCodec(TCodecPtr c0 = nullptr, TCodecPtr c1 = nullptr, TCodecPtr c2 = nullptr, TCodecPtr c3 = nullptr) { - MyTraits.PreservesPrefixGrouping = true; - AddCodec(c0); - AddCodec(c1); - AddCodec(c2); - AddCodec(c3); - } - - TString GetName() const override { - return MyName; - } - - ui8 Encode(TStringBuf in, TBuffer& out) const override; - void Decode(TStringBuf in, TBuffer& out) const override; - - public: - /* + } + }; + + class TNullCodec: public ICodec { + public: + TNullCodec() { + MyTraits.Irreversible = true; + MyTraits.SizeOnDecodeMultiplier = 0; + MyTraits.SizeOnEncodeMultiplier = 0; + } + + TString GetName() const override { + return "null"; + } + + ui8 Encode(TStringBuf, TBuffer& out) const override { + out.Clear(); + return 0; + } + + void Decode(TStringBuf, TBuffer& out) const override { + out.Clear(); + } + + protected: + void DoLearn(ISequenceReader&) override { + } + }; + + class TPipelineCodec: public ICodec { + typedef TVector<TCodecPtr> TPipeline; + + TPipeline Pipeline; + TString MyName; + + public: + explicit TPipelineCodec(TCodecPtr c0 = nullptr, TCodecPtr c1 = nullptr, TCodecPtr c2 = nullptr, TCodecPtr c3 = nullptr) { + MyTraits.PreservesPrefixGrouping = true; + AddCodec(c0); + AddCodec(c1); + AddCodec(c2); + AddCodec(c3); + } + + TString GetName() const override { + return MyName; + } + + ui8 Encode(TStringBuf in, TBuffer& out) const override; + void Decode(TStringBuf in, TBuffer& out) const override; + + public: + /* * Add codecs in the following order: * uncompressed -> codec0 | codec1 | ... | codecN -> compressed */ - TPipelineCodec& AddCodec(TCodecPtr codec); + TPipelineCodec& AddCodec(TCodecPtr codec); - bool AlreadyTrained() const override; - void SetTrained(bool t) override; + bool AlreadyTrained() const override; + void SetTrained(bool t) override; - protected: - void DoLearn(ISequenceReader& in) override { - DoLearnX(in, 1); - } + protected: + void DoLearn(ISequenceReader& in) override { + DoLearnX(in, 1); + } - void DoLearnX(ISequenceReader& in, double sampleSizeMult) override; - void Save(IOutputStream* out) const override; - void Load(IInputStream* in) override; - }; + void DoLearnX(ISequenceReader& in, double sampleSizeMult) override; + void Save(IOutputStream* out) const override; + void Load(IInputStream* in) override; + }; } diff --git a/library/cpp/codecs/codecs_registry.cpp b/library/cpp/codecs/codecs_registry.cpp index 17d07062ab..c8941ec337 100644 --- a/library/cpp/codecs/codecs_registry.cpp +++ b/library/cpp/codecs/codecs_registry.cpp @@ -42,7 +42,7 @@ namespace NCodecs { } else { TPipelineCodec* pipe = new TPipelineCodec; - do { + do { TStringBuf v = name.NextTok(':'); pipe->AddCodec(GetCodec(v)); } while (name); @@ -64,7 +64,7 @@ namespace NCodecs { return vs; } - struct TSolarCodecFactory : ICodecFactory { + struct TSolarCodecFactory : ICodecFactory { TCodecPtr MakeCodec(TStringBuf name) const override { if (TSolarCodec::MyNameShortInt() == name) { return new TSolarCodecShortInt(); @@ -79,7 +79,7 @@ namespace NCodecs { } } - template <class TCodecCls> + template <class TCodecCls> TCodecPtr MakeCodecImpl(const TStringBuf& name, const TStringBuf& type) const { if (TStringBuf("-8k") == type) { return new TCodecCls(1 << 13); @@ -117,7 +117,7 @@ namespace NCodecs { } }; - struct TZStdDictCodecFactory : ICodecFactory { + struct TZStdDictCodecFactory : ICodecFactory { TCodecPtr MakeCodec(TStringBuf name) const override { return new TZStdDictCodec(TZStdDictCodec::ParseCompressionName(name)); } @@ -127,7 +127,7 @@ namespace NCodecs { } }; - struct TCompTableCodecFactory : ICodecFactory { + struct TCompTableCodecFactory : ICodecFactory { TCodecPtr MakeCodec(TStringBuf name) const override { if (TCompTableCodec::MyNameHQ() == name) { return new TCompTableCodec(TCompTableCodec::Q_HIGH); @@ -147,11 +147,11 @@ namespace NCodecs { } }; - struct TBlockCodec : ICodec { + struct TBlockCodec : ICodec { const NBlockCodecs::ICodec* Codec; TBlockCodec(TStringBuf name) - : Codec(NBlockCodecs::Codec(name)) + : Codec(NBlockCodecs::Codec(name)) { } @@ -174,11 +174,11 @@ namespace NCodecs { } }; - struct TBlockCodecsFactory : ICodecFactory { + struct TBlockCodecsFactory : ICodecFactory { using TRegistry = THashMap<TString, TCodecPtr>; TRegistry Registry; - TBlockCodecsFactory() { + TBlockCodecsFactory() { for (TStringBuf codec : NBlockCodecs::ListAllCodecs()) { Register(codec); } @@ -205,12 +205,12 @@ namespace NCodecs { } }; - TCodecRegistry::TCodecRegistry() { + TCodecRegistry::TCodecRegistry() { RegisterFactory(new TInstanceFactory<TTrivialCodec>); RegisterFactory(new TInstanceFactory<TTrivialTrainableCodec>); RegisterFactory(new TInstanceFactory<THuffmanCodec>); - RegisterFactory(new TInstanceFactory<TPForCodec<ui64, true>>); - RegisterFactory(new TInstanceFactory<TPForCodec<ui32, true>>); + RegisterFactory(new TInstanceFactory<TPForCodec<ui64, true>>); + RegisterFactory(new TInstanceFactory<TPForCodec<ui32, true>>); RegisterFactory(new TSolarCodecFactory); RegisterFactory(new TZStdDictCodecFactory); RegisterFactory(new TCompTableCodecFactory); diff --git a/library/cpp/codecs/codecs_registry.h b/library/cpp/codecs/codecs_registry.h index 53710310d5..abd4a38cc5 100644 --- a/library/cpp/codecs/codecs_registry.h +++ b/library/cpp/codecs/codecs_registry.h @@ -4,13 +4,13 @@ #include <util/string/cast.h> namespace NCodecs { - struct TNoCodecException : TCodecException { - TNoCodecException(TStringBuf name) { + struct TNoCodecException : TCodecException { + TNoCodecException(TStringBuf name) { (*this) << "unknown codec: " << name; } }; - struct ICodecFactory : TAtomicRefCount<ICodecFactory> { + struct ICodecFactory : TAtomicRefCount<ICodecFactory> { virtual ~ICodecFactory() = default; virtual TCodecPtr MakeCodec(TStringBuf name) const = 0; virtual TVector<TString> ListNames() const = 0; @@ -19,8 +19,8 @@ namespace NCodecs { typedef TIntrusivePtr<ICodecFactory> TCodecFactoryPtr; namespace NPrivate { - template <typename TCodec> - struct TInstanceFactory : ICodecFactory { + template <typename TCodec> + struct TInstanceFactory : ICodecFactory { TCodecPtr MakeCodec(TStringBuf) const override { return new TCodec; } @@ -52,7 +52,7 @@ namespace NCodecs { void RegisterCodecFactory(TCodecFactoryPtr fact); - template <typename TCodec> + template <typename TCodec> void RegisterCodec() { RegisterCodecFactory(new NPrivate::TInstanceFactory<TCodec>()); } diff --git a/library/cpp/codecs/comptable_codec.cpp b/library/cpp/codecs/comptable_codec.cpp index 476b8ada80..cf747121ba 100644 --- a/library/cpp/codecs/comptable_codec.cpp +++ b/library/cpp/codecs/comptable_codec.cpp @@ -4,12 +4,12 @@ #include <util/string/cast.h> namespace NCodecs { - class TCompTableCodec::TImpl: public TAtomicRefCount<TImpl> { + class TCompTableCodec::TImpl: public TAtomicRefCount<TImpl> { public: TImpl(EQuality q) : Quality(q) - { - } + { + } void Init() { Compressor.Reset(new NCompTable::TChunkCompressor{(bool)Quality, Table}); diff --git a/library/cpp/codecs/comptable_codec.h b/library/cpp/codecs/comptable_codec.h index 7ba4f4c543..d0f4361780 100644 --- a/library/cpp/codecs/comptable_codec.h +++ b/library/cpp/codecs/comptable_codec.h @@ -5,36 +5,36 @@ #include <util/generic/ptr.h> namespace NCodecs { - class TCompTableCodec: public ICodec { - class TImpl; - TIntrusivePtr<TImpl> Impl; + class TCompTableCodec: public ICodec { + class TImpl; + TIntrusivePtr<TImpl> Impl; - public: - enum EQuality { - Q_LOW = 0, - Q_HIGH = 1 - }; + public: + enum EQuality { + Q_LOW = 0, + Q_HIGH = 1 + }; - explicit TCompTableCodec(EQuality q = Q_HIGH); - ~TCompTableCodec() override; + explicit TCompTableCodec(EQuality q = Q_HIGH); + ~TCompTableCodec() override; - static TStringBuf MyNameHQ() { - return "comptable-hq"; - } - static TStringBuf MyNameLQ() { - return "comptable-lq"; - } + static TStringBuf MyNameHQ() { + return "comptable-hq"; + } + static TStringBuf MyNameLQ() { + return "comptable-lq"; + } - TString GetName() const override; + TString GetName() const override; - ui8 Encode(TStringBuf in, TBuffer& out) const override; + ui8 Encode(TStringBuf in, TBuffer& out) const override; - void Decode(TStringBuf in, TBuffer& out) const override; + void Decode(TStringBuf in, TBuffer& out) const override; - protected: - void DoLearn(ISequenceReader& in) override; - void Save(IOutputStream* out) const override; - void Load(IInputStream* in) override; - }; + protected: + void DoLearn(ISequenceReader& in) override; + void Save(IOutputStream* out) const override; + void Load(IInputStream* in) override; + }; } diff --git a/library/cpp/codecs/delta_codec.cpp b/library/cpp/codecs/delta_codec.cpp index 61606d6f6f..b9ed146dcb 100644 --- a/library/cpp/codecs/delta_codec.cpp +++ b/library/cpp/codecs/delta_codec.cpp @@ -1,21 +1,21 @@ #include "delta_codec.h" namespace NCodecs { - template <> - TStringBuf TDeltaCodec<ui64, true>::MyName() { - return "delta64-unsigned"; - } - template <> - TStringBuf TDeltaCodec<ui32, true>::MyName() { - return "delta32-unsigned"; - } - template <> - TStringBuf TDeltaCodec<ui64, false>::MyName() { - return "delta64-signed"; - } - template <> - TStringBuf TDeltaCodec<ui32, false>::MyName() { - return "delta32-signed"; - } + template <> + TStringBuf TDeltaCodec<ui64, true>::MyName() { + return "delta64-unsigned"; + } + template <> + TStringBuf TDeltaCodec<ui32, true>::MyName() { + return "delta32-unsigned"; + } + template <> + TStringBuf TDeltaCodec<ui64, false>::MyName() { + return "delta64-signed"; + } + template <> + TStringBuf TDeltaCodec<ui32, false>::MyName() { + return "delta32-signed"; + } } diff --git a/library/cpp/codecs/delta_codec.h b/library/cpp/codecs/delta_codec.h index 21325825e6..4e5dbb8f75 100644 --- a/library/cpp/codecs/delta_codec.h +++ b/library/cpp/codecs/delta_codec.h @@ -8,136 +8,136 @@ #include <util/string/cast.h> namespace NCodecs { - template <typename T = ui64, bool UnsignedDelta = true> - class TDeltaCodec: public ICodec { - static_assert(std::is_integral<T>::value, "expect std::is_integral<T>::value"); - - public: - using TUnsigned = std::make_unsigned_t<T>; - using TSigned = std::make_signed_t<T>; - using TDelta = std::conditional_t<UnsignedDelta, TUnsigned, TSigned>; - - private: - const TDelta MinDelta{Min<TDelta>()}; - const TDelta MaxDelta{Max<TDelta>() - 1}; - const TDelta InvalidDelta{MaxDelta + 1}; - - Y_FORCE_INLINE static TDelta AddSafe(TUnsigned a, TUnsigned b) { - return a + b; - } - - Y_FORCE_INLINE static TDelta SubSafe(TUnsigned a, TUnsigned b) { - return a - b; - } - - public: - struct TDecoder { - const TDelta InvalidDelta{Max<TDelta>()}; - - T Last = 0; - T Result = 0; - - bool First = true; - bool Invalid = false; - - Y_FORCE_INLINE bool Decode(TDelta t) { - if (Y_UNLIKELY(First)) { - First = false; - Result = Last = t; - return true; - } - - if (Y_UNLIKELY(Invalid)) { - Invalid = false; - Last = 0; - Result = t; - return true; - } - - Result = (Last += t); - Invalid = t == InvalidDelta; - - return !Invalid; + template <typename T = ui64, bool UnsignedDelta = true> + class TDeltaCodec: public ICodec { + static_assert(std::is_integral<T>::value, "expect std::is_integral<T>::value"); + + public: + using TUnsigned = std::make_unsigned_t<T>; + using TSigned = std::make_signed_t<T>; + using TDelta = std::conditional_t<UnsignedDelta, TUnsigned, TSigned>; + + private: + const TDelta MinDelta{Min<TDelta>()}; + const TDelta MaxDelta{Max<TDelta>() - 1}; + const TDelta InvalidDelta{MaxDelta + 1}; + + Y_FORCE_INLINE static TDelta AddSafe(TUnsigned a, TUnsigned b) { + return a + b; + } + + Y_FORCE_INLINE static TDelta SubSafe(TUnsigned a, TUnsigned b) { + return a - b; + } + + public: + struct TDecoder { + const TDelta InvalidDelta{Max<TDelta>()}; + + T Last = 0; + T Result = 0; + + bool First = true; + bool Invalid = false; + + Y_FORCE_INLINE bool Decode(TDelta t) { + if (Y_UNLIKELY(First)) { + First = false; + Result = Last = t; + return true; + } + + if (Y_UNLIKELY(Invalid)) { + Invalid = false; + Last = 0; + Result = t; + return true; + } + + Result = (Last += t); + Invalid = t == InvalidDelta; + + return !Invalid; } - }; + }; - public: - static TStringBuf MyName(); + public: + static TStringBuf MyName(); - TDeltaCodec() { - MyTraits.SizeOfInputElement = sizeof(T); - MyTraits.AssumesStructuredInput = true; + TDeltaCodec() { + MyTraits.SizeOfInputElement = sizeof(T); + MyTraits.AssumesStructuredInput = true; } - TString GetName() const override { + TString GetName() const override { return ToString(MyName()); - } + } - template <class TItem> - static void AppendTo(TBuffer& b, TItem t) { - b.Append((char*)&t, sizeof(t)); - } + template <class TItem> + static void AppendTo(TBuffer& b, TItem t) { + b.Append((char*)&t, sizeof(t)); + } - ui8 Encode(TStringBuf s, TBuffer& b) const override { - b.Clear(); + ui8 Encode(TStringBuf s, TBuffer& b) const override { + b.Clear(); if (s.empty()) { - return 0; - } + return 0; + } b.Reserve(s.size()); TArrayRef<const T> tin{(const T*)s.data(), s.size() / sizeof(T)}; const T* it = tin.begin(); - TDelta last = *(it++); - AppendTo(b, last); + TDelta last = *(it++); + AppendTo(b, last); - TDelta maxt = SubSafe(MaxDelta, last); - TDelta mint = AddSafe(MinDelta, last); + TDelta maxt = SubSafe(MaxDelta, last); + TDelta mint = AddSafe(MinDelta, last); for (; it != tin.end(); ++it) { - TDelta t = *it; - - if (Y_LIKELY((t >= mint) & (t <= maxt))) { - AppendTo(b, t - last); - last = t; - maxt = SubSafe(MaxDelta, last); - mint = AddSafe(MinDelta, last); - } else { - // delta overflow - AppendTo(b, InvalidDelta); - AppendTo(b, t); - last = 0; - maxt = MaxDelta; - mint = MinDelta; - } - } - - return 0; - } - - void Decode(TStringBuf s, TBuffer& b) const override { - b.Clear(); + TDelta t = *it; + + if (Y_LIKELY((t >= mint) & (t <= maxt))) { + AppendTo(b, t - last); + last = t; + maxt = SubSafe(MaxDelta, last); + mint = AddSafe(MinDelta, last); + } else { + // delta overflow + AppendTo(b, InvalidDelta); + AppendTo(b, t); + last = 0; + maxt = MaxDelta; + mint = MinDelta; + } + } + + return 0; + } + + void Decode(TStringBuf s, TBuffer& b) const override { + b.Clear(); if (s.empty()) { - return; + return; } b.Reserve(s.size()); TArrayRef<const T> tin{(const T*)s.data(), s.size() / sizeof(T)}; - TDecoder dec; + TDecoder dec; for (const T* it = tin.begin(); it != tin.end(); ++it) { - T tmp; - memcpy(&tmp, it, sizeof(tmp)); - if (dec.Decode(tmp)) { - AppendTo(b, dec.Result); - } + T tmp; + memcpy(&tmp, it, sizeof(tmp)); + if (dec.Decode(tmp)) { + AppendTo(b, dec.Result); + } } } - protected: - void DoLearn(ISequenceReader&) override { - } - }; + protected: + void DoLearn(ISequenceReader&) override { + } + }; } diff --git a/library/cpp/codecs/float_huffman.cpp b/library/cpp/codecs/float_huffman.cpp index c4a8bd228f..a95ca5b41d 100644 --- a/library/cpp/codecs/float_huffman.cpp +++ b/library/cpp/codecs/float_huffman.cpp @@ -55,7 +55,7 @@ namespace NCodecs::NFloatHuff { {0x3c000000, 0x12, 5, 24}, // [0.0078125, 0.03125), 29 bits, prefix [01001] {0x3b000000, 0x26, 6, 34}, // [0.001953125, end of range), 40 bits, prefix [011001] {0x00000000, 0x16, 5, 32}, // whole range, 37 bits, prefix [01101] - }; + }; [[noreturn]] Y_NO_INLINE void ThrowInvalidOffset(size_t size, size_t byteOffset) { ythrow yexception() << diff --git a/library/cpp/codecs/greedy_dict/gd_builder.cpp b/library/cpp/codecs/greedy_dict/gd_builder.cpp index 561bfbca01..802c721753 100644 --- a/library/cpp/codecs/greedy_dict/gd_builder.cpp +++ b/library/cpp/codecs/greedy_dict/gd_builder.cpp @@ -9,134 +9,134 @@ #include <util/system/rusage.h> namespace NGreedyDict { - void TDictBuilder::RebuildCounts(ui32 maxcand, bool final) { - if (!Current) { + void TDictBuilder::RebuildCounts(ui32 maxcand, bool final) { + if (!Current) { Current = MakeHolder<TEntrySet>(); - Current->InitWithAlpha(); - } + Current->InitWithAlpha(); + } - TEntrySet& set = *Current; + TEntrySet& set = *Current; - for (auto& it : set) - it.Count = 0; + for (auto& it : set) + it.Count = 0; - CompoundCounts = nullptr; - CompoundCountsPool.Clear(); + CompoundCounts = nullptr; + CompoundCountsPool.Clear(); - if (!final) { + if (!final) { CompoundCounts = MakeHolder<TCompoundCounts>(&CompoundCountsPool); - CompoundCounts->reserve(maxcand); - } + CompoundCounts->reserve(maxcand); + } - Shuffle(Input.begin(), Input.end(), Rng); + Shuffle(Input.begin(), Input.end(), Rng); - for (auto str : Input) { - if (!final && CompoundCounts->size() > maxcand) - break; + for (auto str : Input) { + if (!final && CompoundCounts->size() > maxcand) + break; - i32 prev = -1; + i32 prev = -1; - while (!!str) { - TEntry* e = set.FindPrefix(str); - ui32 num = e->Number; + while (!!str) { + TEntry* e = set.FindPrefix(str); + ui32 num = e->Number; - e->Count += 1; - if (!final && prev >= 0) { - (*CompoundCounts)[Compose(prev, num)] += 1; - } + e->Count += 1; + if (!final && prev >= 0) { + (*CompoundCounts)[Compose(prev, num)] += 1; + } - prev = num; - ++set.TotalCount; + prev = num; + ++set.TotalCount; } - } + } - Current->SetModelP(); + Current->SetModelP(); } - ui32 TDictBuilder::BuildNextGeneration(ui32 maxent) { - TAutoPtr<TEntrySet> newset = new TEntrySet; - newset->InitWithAlpha(); - maxent -= newset->size(); - - ui32 additions = 0; - ui32 deletions = 0; - - { - const TEntrySet& set = *Current; - - Candidates.clear(); - const ui32 total = set.TotalCount; - const float minpval = Settings.MinPValue; - const EEntryStatTest test = Settings.StatTest; - const EEntryScore score = Settings.Score; - const ui32 mincnt = Settings.MinAbsCount; - - for (const auto& it : set) { - const TEntry& e = it; - float modelp = e.ModelP; - ui32 cnt = e.Count; - - if (e.HasPrefix() && e.Count > mincnt && StatTest(test, modelp, cnt, total) > minpval) - Candidates.push_back(TCandidate(-Score(score, e.Len(), modelp, cnt, total), it.Number)); - } - - if (!!CompoundCounts) { - for (TCompoundCounts::const_iterator it = CompoundCounts->begin(); it != CompoundCounts->end(); ++it) { - const TEntry& prev = set.Get(Prev(it->first)); - const TEntry& next = set.Get(Next(it->first)); - float modelp = ModelP(prev.Count, next.Count, total); - ui32 cnt = it->second; - if (cnt > mincnt && StatTest(test, modelp, cnt, total) > minpval) - Candidates.push_back(TCandidate(-Score(score, prev.Len() + next.Len(), modelp, cnt, total), it->first)); - } + ui32 TDictBuilder::BuildNextGeneration(ui32 maxent) { + TAutoPtr<TEntrySet> newset = new TEntrySet; + newset->InitWithAlpha(); + maxent -= newset->size(); + + ui32 additions = 0; + ui32 deletions = 0; + + { + const TEntrySet& set = *Current; + + Candidates.clear(); + const ui32 total = set.TotalCount; + const float minpval = Settings.MinPValue; + const EEntryStatTest test = Settings.StatTest; + const EEntryScore score = Settings.Score; + const ui32 mincnt = Settings.MinAbsCount; + + for (const auto& it : set) { + const TEntry& e = it; + float modelp = e.ModelP; + ui32 cnt = e.Count; + + if (e.HasPrefix() && e.Count > mincnt && StatTest(test, modelp, cnt, total) > minpval) + Candidates.push_back(TCandidate(-Score(score, e.Len(), modelp, cnt, total), it.Number)); + } + + if (!!CompoundCounts) { + for (TCompoundCounts::const_iterator it = CompoundCounts->begin(); it != CompoundCounts->end(); ++it) { + const TEntry& prev = set.Get(Prev(it->first)); + const TEntry& next = set.Get(Next(it->first)); + float modelp = ModelP(prev.Count, next.Count, total); + ui32 cnt = it->second; + if (cnt > mincnt && StatTest(test, modelp, cnt, total) > minpval) + Candidates.push_back(TCandidate(-Score(score, prev.Len() + next.Len(), modelp, cnt, total), it->first)); + } } - Sort(Candidates.begin(), Candidates.end()); + Sort(Candidates.begin(), Candidates.end()); - if (Candidates.size() > maxent) - Candidates.resize(maxent); + if (Candidates.size() > maxent) + Candidates.resize(maxent); - for (const auto& candidate : Candidates) { - if (IsCompound(candidate.second)) { - additions++; - newset->Add(set.Get(Prev(candidate.second)).Str, set.Get(Next(candidate.second)).Str); - } else { - newset->Add(set.Get(candidate.second).Str); - } + for (const auto& candidate : Candidates) { + if (IsCompound(candidate.second)) { + additions++; + newset->Add(set.Get(Prev(candidate.second)).Str, set.Get(Next(candidate.second)).Str); + } else { + newset->Add(set.Get(candidate.second).Str); + } } - - deletions = set.size() - (newset->size() - additions); + + deletions = set.size() - (newset->size() - additions); } - Current = newset; - Current->BuildHierarchy(); - return deletions + additions; + Current = newset; + Current->BuildHierarchy(); + return deletions + additions; } - ui32 TDictBuilder::Build(ui32 maxentries, ui32 maxiters, ui32 mindiff) { - size_t totalsz = 0; - for (auto it : Input) + ui32 TDictBuilder::Build(ui32 maxentries, ui32 maxiters, ui32 mindiff) { + size_t totalsz = 0; + for (auto it : Input) totalsz += it.size(); - while (maxiters) { - maxiters--; + while (maxiters) { + maxiters--; - RebuildCounts(maxentries * Settings.GrowLimit, false); + RebuildCounts(maxentries * Settings.GrowLimit, false); - if (Settings.Verbose) { - TString mess = Sprintf("iter:%" PRIu32 " sz:%" PRIu32 " pend:%" PRIu32, maxiters, (ui32)Current->size(), (ui32)CompoundCounts->size()); + if (Settings.Verbose) { + TString mess = Sprintf("iter:%" PRIu32 " sz:%" PRIu32 " pend:%" PRIu32, maxiters, (ui32)Current->size(), (ui32)CompoundCounts->size()); Clog << Sprintf("%-110s RSS=%" PRIu32 "M", mess.data(), (ui32)(TRusage::Get().MaxRss >> 20)) << Endl; - } - - ui32 diff = BuildNextGeneration(maxentries); + } - if (Current->size() == maxentries && diff < mindiff) - break; + ui32 diff = BuildNextGeneration(maxentries); + + if (Current->size() == maxentries && diff < mindiff) + break; } - RebuildCounts(0, true); - Current->SetScores(Settings.Score); - return maxiters; + RebuildCounts(0, true); + Current->SetScores(Settings.Score); + return maxiters; } } diff --git a/library/cpp/codecs/greedy_dict/gd_builder.h b/library/cpp/codecs/greedy_dict/gd_builder.h index b8e9a5e37b..ab0057e1ca 100644 --- a/library/cpp/codecs/greedy_dict/gd_builder.h +++ b/library/cpp/codecs/greedy_dict/gd_builder.h @@ -6,89 +6,89 @@ #include <util/random/fast.h> namespace NGreedyDict { - struct TBuildSettings { - EEntryStatTest StatTest = EST_SIMPLE_NORM; - EEntryScore Score = ES_LEN_SIMPLE; - - float MinPValue = 0.75; - ui32 MinAbsCount = 10; - ui32 GrowLimit = 10; // times of maxentries - bool Verbose = false; - }; - - class TDictBuilder { - using TCompoundCounts = THashMap<ui64, ui32, THash<ui64>, TEqualTo<ui64>, TPoolAllocator>; - using TCandidate = std::pair<float, ui64>; - using TCandidates = TVector<TCandidate>; - - private: - TFastRng64 Rng{0x1a5d0ac170565c1c, 0x0be7bc27, 0x6235f6f57820aa0d, 0xafdc7fb}; - TStringBufs Input; - - THolder<TEntrySet> Current; - - TMemoryPool CompoundCountsPool; - THolder<TCompoundCounts> CompoundCounts; - - TCandidates Candidates; - - TBuildSettings Settings; - - public: - TDictBuilder(const TBuildSettings& s = TBuildSettings()) - : CompoundCountsPool(8112, TMemoryPool::TLinearGrow::Instance()) - , Settings(s) - { - } - - void SetInput(const TStringBufs& in) { - Input = in; - } - - const TBuildSettings& GetSettings() const { - return Settings; - } - - TBuildSettings& GetSettings() { - return Settings; - } - - void SetSettings(const TBuildSettings& s) { - Settings = s; - } - - TEntrySet& EntrySet() { - return *Current; - } - - const TEntrySet& EntrySet() const { - return *Current; - } - - THolder<TEntrySet> ReleaseEntrySet() { - return std::move(Current); - } - - ui32 /*iters*/ Build(ui32 maxentries, ui32 maxiters = 16, ui32 mindiff = 10); - - public: - void RebuildCounts(ui32 maxcand, bool final); - ui32 /*diff size*/ BuildNextGeneration(ui32 maxent); - - static bool IsCompound(ui64 ent) { - return ent & 0xFFFFFFFF00000000ULL; - } - - static ui32 Next(ui64 ent) { - return ent; - } - static ui32 Prev(ui64 ent) { - return (ent >> 32) - 1; - } - - static ui64 Compose(ui32 prev, ui32 next) { - return ((prev + 1ULL) << 32) | next; - } - }; + struct TBuildSettings { + EEntryStatTest StatTest = EST_SIMPLE_NORM; + EEntryScore Score = ES_LEN_SIMPLE; + + float MinPValue = 0.75; + ui32 MinAbsCount = 10; + ui32 GrowLimit = 10; // times of maxentries + bool Verbose = false; + }; + + class TDictBuilder { + using TCompoundCounts = THashMap<ui64, ui32, THash<ui64>, TEqualTo<ui64>, TPoolAllocator>; + using TCandidate = std::pair<float, ui64>; + using TCandidates = TVector<TCandidate>; + + private: + TFastRng64 Rng{0x1a5d0ac170565c1c, 0x0be7bc27, 0x6235f6f57820aa0d, 0xafdc7fb}; + TStringBufs Input; + + THolder<TEntrySet> Current; + + TMemoryPool CompoundCountsPool; + THolder<TCompoundCounts> CompoundCounts; + + TCandidates Candidates; + + TBuildSettings Settings; + + public: + TDictBuilder(const TBuildSettings& s = TBuildSettings()) + : CompoundCountsPool(8112, TMemoryPool::TLinearGrow::Instance()) + , Settings(s) + { + } + + void SetInput(const TStringBufs& in) { + Input = in; + } + + const TBuildSettings& GetSettings() const { + return Settings; + } + + TBuildSettings& GetSettings() { + return Settings; + } + + void SetSettings(const TBuildSettings& s) { + Settings = s; + } + + TEntrySet& EntrySet() { + return *Current; + } + + const TEntrySet& EntrySet() const { + return *Current; + } + + THolder<TEntrySet> ReleaseEntrySet() { + return std::move(Current); + } + + ui32 /*iters*/ Build(ui32 maxentries, ui32 maxiters = 16, ui32 mindiff = 10); + + public: + void RebuildCounts(ui32 maxcand, bool final); + ui32 /*diff size*/ BuildNextGeneration(ui32 maxent); + + static bool IsCompound(ui64 ent) { + return ent & 0xFFFFFFFF00000000ULL; + } + + static ui32 Next(ui64 ent) { + return ent; + } + static ui32 Prev(ui64 ent) { + return (ent >> 32) - 1; + } + + static ui64 Compose(ui32 prev, ui32 next) { + return ((prev + 1ULL) << 32) | next; + } + }; } diff --git a/library/cpp/codecs/greedy_dict/gd_entry.cpp b/library/cpp/codecs/greedy_dict/gd_entry.cpp index 2c315c7f7c..0603a9fca8 100644 --- a/library/cpp/codecs/greedy_dict/gd_entry.cpp +++ b/library/cpp/codecs/greedy_dict/gd_entry.cpp @@ -5,94 +5,94 @@ #include <util/generic/singleton.h> namespace NGreedyDict { - class TAlphas { - char Memory[512]; - - public: - TStringBufs Alphas; - - TAlphas() { - for (ui32 i = 0; i < 256; ++i) { - Memory[2 * i] = (char)i; - Memory[2 * i + 1] = 0; - - Alphas.push_back(TStringBuf(&Memory[2 * i], 1)); - } + class TAlphas { + char Memory[512]; + + public: + TStringBufs Alphas; + + TAlphas() { + for (ui32 i = 0; i < 256; ++i) { + Memory[2 * i] = (char)i; + Memory[2 * i + 1] = 0; + + Alphas.push_back(TStringBuf(&Memory[2 * i], 1)); + } + } + }; + + void TEntrySet::InitWithAlpha() { + Pool.ClearKeepFirstChunk(); + const TStringBufs& a = Singleton<TAlphas>()->Alphas; + for (auto it : a) { + Add(it); } - }; - - void TEntrySet::InitWithAlpha() { - Pool.ClearKeepFirstChunk(); - const TStringBufs& a = Singleton<TAlphas>()->Alphas; - for (auto it : a) { - Add(it); - } - BuildHierarchy(); + BuildHierarchy(); } - void TEntrySet::BuildHierarchy() { - Sort(begin(), end(), TEntry::StrLess); + void TEntrySet::BuildHierarchy() { + Sort(begin(), end(), TEntry::StrLess); - TCompactTrieBuilder<char, ui32, TAsIsPacker<ui32>> builder(CTBF_PREFIX_GROUPED); + TCompactTrieBuilder<char, ui32, TAsIsPacker<ui32>> builder(CTBF_PREFIX_GROUPED); - for (iterator it = begin(); it != end(); ++it) { - it->Number = (it - begin()); - TStringBuf suff = it->Str; - size_t len = 0; - ui32 val = 0; + for (iterator it = begin(); it != end(); ++it) { + it->Number = (it - begin()); + TStringBuf suff = it->Str; + size_t len = 0; + ui32 val = 0; if (builder.FindLongestPrefix(suff.data(), suff.size(), &len, &val) && len) { - it->NearestPrefix = val; - } + it->NearestPrefix = val; + } builder.Add(suff.data(), suff.size(), it->Number); } - TBufferOutput bout; - builder.Save(bout); - Trie.Init(TBlob::FromBuffer(bout.Buffer())); + TBufferOutput bout; + builder.Save(bout); + Trie.Init(TBlob::FromBuffer(bout.Buffer())); } - TEntry* TEntrySet::FindPrefix(TStringBuf& str) { - size_t len = 0; - ui32 off = 0; + TEntry* TEntrySet::FindPrefix(TStringBuf& str) { + size_t len = 0; + ui32 off = 0; - if (!Trie.FindLongestPrefix(str, &len, &off)) { - return nullptr; - } + if (!Trie.FindLongestPrefix(str, &len, &off)) { + return nullptr; + } - str.Skip(len); - return &Get(off); + str.Skip(len); + return &Get(off); } - void TEntrySet::SetModelP() { - for (iterator it = begin(); it != end(); ++it) { - TEntry& e = *it; + void TEntrySet::SetModelP() { + for (iterator it = begin(); it != end(); ++it) { + TEntry& e = *it; - if (!e.HasPrefix()) { - e.ModelP = 0; - continue; - } + if (!e.HasPrefix()) { + e.ModelP = 0; + continue; + } - TStringBuf suff = e.Str; - const TEntry& p = Get(e.NearestPrefix); - suff.Skip(p.Len()); + TStringBuf suff = e.Str; + const TEntry& p = Get(e.NearestPrefix); + suff.Skip(p.Len()); - float modelp = float(p.Count + e.Count) / TotalCount; + float modelp = float(p.Count + e.Count) / TotalCount; - while (!!suff) { - TEntry* pp = FindPrefix(suff); - modelp *= float(pp->Count + e.Count) / TotalCount; - } + while (!!suff) { + TEntry* pp = FindPrefix(suff); + modelp *= float(pp->Count + e.Count) / TotalCount; + } - e.ModelP = modelp; + e.ModelP = modelp; } } - void TEntrySet::SetScores(EEntryScore s) { - for (auto& it : *this) { - it.Score = Score(s, it.Len(), it.ModelP, it.Count, TotalCount); - } + void TEntrySet::SetScores(EEntryScore s) { + for (auto& it : *this) { + it.Score = Score(s, it.Len(), it.ModelP, it.Count, TotalCount); + } } } diff --git a/library/cpp/codecs/greedy_dict/gd_entry.h b/library/cpp/codecs/greedy_dict/gd_entry.h index 18b5be0e15..e123c66b4a 100644 --- a/library/cpp/codecs/greedy_dict/gd_entry.h +++ b/library/cpp/codecs/greedy_dict/gd_entry.h @@ -11,93 +11,93 @@ #include <util/memory/pool.h> namespace NGreedyDict { - using TStringBufs = TVector<TStringBuf>; + using TStringBufs = TVector<TStringBuf>; - struct TEntry { - static const i32 NoPrefix = -1; + struct TEntry { + static const i32 NoPrefix = -1; - TStringBuf Str; + TStringBuf Str; - i32 NearestPrefix = NoPrefix; - ui32 Count = 0; - ui32 Number = 0; - float ModelP = 0; - float Score = 0; + i32 NearestPrefix = NoPrefix; + ui32 Count = 0; + ui32 Number = 0; + float ModelP = 0; + float Score = 0; - TEntry(TStringBuf b = TStringBuf(), ui32 cnt = 0) - : Str(b) - , Count(cnt) - { - } + TEntry(TStringBuf b = TStringBuf(), ui32 cnt = 0) + : Str(b) + , Count(cnt) + { + } - bool HasPrefix() const { - return NearestPrefix != NoPrefix; - } - ui32 Len() const { + bool HasPrefix() const { + return NearestPrefix != NoPrefix; + } + ui32 Len() const { return Str.size(); - } + } - static bool StrLess(const TEntry& a, const TEntry& b) { - return a.Str < b.Str; - } - static bool NumberLess(const TEntry& a, const TEntry& b) { - return a.Number < b.Number; - } - static bool ScoreMore(const TEntry& a, const TEntry& b) { - return a.Score > b.Score; - } - }; + static bool StrLess(const TEntry& a, const TEntry& b) { + return a.Str < b.Str; + } + static bool NumberLess(const TEntry& a, const TEntry& b) { + return a.Number < b.Number; + } + static bool ScoreMore(const TEntry& a, const TEntry& b) { + return a.Score > b.Score; + } + }; - class TEntrySet: public TVector<TEntry>, TNonCopyable { - TMemoryPool Pool{8112}; - TCompactTrie<char, ui32, TAsIsPacker<ui32>> Trie; + class TEntrySet: public TVector<TEntry>, TNonCopyable { + TMemoryPool Pool{8112}; + TCompactTrie<char, ui32, TAsIsPacker<ui32>> Trie; - public: - ui32 TotalCount = 0; + public: + ui32 TotalCount = 0; - void InitWithAlpha(); + void InitWithAlpha(); - void Add(TStringBuf a) { + void Add(TStringBuf a) { push_back(TStringBuf(Pool.Append(a.data(), a.size()), a.size())); - } + } - void Add(TStringBuf a, TStringBuf b) { + void Add(TStringBuf a, TStringBuf b) { size_t sz = a.size() + b.size(); - char* p = (char*)Pool.Allocate(sz); + char* p = (char*)Pool.Allocate(sz); memcpy(p, a.data(), a.size()); memcpy(p + a.size(), b.data(), b.size()); - push_back(TStringBuf(p, sz)); - } + push_back(TStringBuf(p, sz)); + } - TEntry& Get(ui32 idx) { - return (*this)[idx]; - } + TEntry& Get(ui32 idx) { + return (*this)[idx]; + } - const TEntry& Get(ui32 idx) const { - return (*this)[idx]; - } + const TEntry& Get(ui32 idx) const { + return (*this)[idx]; + } - void BuildHierarchy(); + void BuildHierarchy(); - // longest prefix - TEntry* FindPrefix(TStringBuf& str); + // longest prefix + TEntry* FindPrefix(TStringBuf& str); - const TEntry* FindPrefix(TStringBuf& str) const { - return ((TEntrySet*)this)->FindPrefix(str); - } + const TEntry* FindPrefix(TStringBuf& str) const { + return ((TEntrySet*)this)->FindPrefix(str); + } - const TEntry* FirstPrefix(const TEntry& e, TStringBuf& suff) { - if (!e.HasPrefix()) - return nullptr; + const TEntry* FirstPrefix(const TEntry& e, TStringBuf& suff) { + if (!e.HasPrefix()) + return nullptr; - const TEntry& p = Get(e.NearestPrefix); - suff = e.Str; + const TEntry& p = Get(e.NearestPrefix); + suff = e.Str; suff.Skip(p.Str.size()); - return &p; - } + return &p; + } - void SetModelP(); - void SetScores(EEntryScore); - }; + void SetModelP(); + void SetScores(EEntryScore); + }; } diff --git a/library/cpp/codecs/greedy_dict/gd_stats.h b/library/cpp/codecs/greedy_dict/gd_stats.h index b63c4c38d2..90f46a0fb9 100644 --- a/library/cpp/codecs/greedy_dict/gd_stats.h +++ b/library/cpp/codecs/greedy_dict/gd_stats.h @@ -1,78 +1,78 @@ #pragma once -#include <util/generic/ymath.h> +#include <util/generic/ymath.h> #include <util/generic/algorithm.h> #include <util/generic/yexception.h> namespace NGreedyDict { - enum EEntryScore { - ES_COUNT, - ES_LEN_COUNT, - ES_SIMPLE, - ES_LEN_SIMPLE, - ES_SOLAR - }; + enum EEntryScore { + ES_COUNT, + ES_LEN_COUNT, + ES_SIMPLE, + ES_LEN_SIMPLE, + ES_SOLAR + }; - enum EEntryStatTest { - EST_NONE = 0, - EST_SIMPLE_NORM = 2 - }; + enum EEntryStatTest { + EST_NONE = 0, + EST_SIMPLE_NORM = 2 + }; - inline float ModelP(ui32 countA, ui32 countB, ui32 total) { - return float(countA) * countB / total / total; - } + inline float ModelP(ui32 countA, ui32 countB, ui32 total) { + return float(countA) * countB / total / total; + } - // P (ab | dependent) - inline float SimpleTest(float modelp, ui32 countAB, ui32 total) { - float realp = float(countAB) / total; - return modelp >= realp ? 0 : (realp - modelp); - } + // P (ab | dependent) + inline float SimpleTest(float modelp, ui32 countAB, ui32 total) { + float realp = float(countAB) / total; + return modelp >= realp ? 0 : (realp - modelp); + } - inline float SolarTest(float modelp, ui32 countAB, ui32 total) { - float realp = float(countAB) / total; - return modelp >= realp ? 0 : (modelp + realp * (log(realp / modelp) - 1)); - } + inline float SolarTest(float modelp, ui32 countAB, ui32 total) { + float realp = float(countAB) / total; + return modelp >= realp ? 0 : (modelp + realp * (log(realp / modelp) - 1)); + } - // P (ab | dependent) / P (ab) - inline float SimpleTestNorm(float modelp, ui32 countAB, ui32 total) { - float realp = float(countAB) / total; - return modelp >= realp ? 0 : (realp - modelp) / realp; - } + // P (ab | dependent) / P (ab) + inline float SimpleTestNorm(float modelp, ui32 countAB, ui32 total) { + float realp = float(countAB) / total; + return modelp >= realp ? 0 : (realp - modelp) / realp; + } - inline float StatTest(EEntryStatTest test, float modelp, ui32 countAB, ui32 total) { - if (!total) { - return 0; - } - switch (test) { - case EST_NONE: - return 1; - case EST_SIMPLE_NORM: - return SimpleTestNorm(modelp, countAB, total); - } - Y_FAIL("no way!"); + inline float StatTest(EEntryStatTest test, float modelp, ui32 countAB, ui32 total) { + if (!total) { + return 0; + } + switch (test) { + case EST_NONE: + return 1; + case EST_SIMPLE_NORM: + return SimpleTestNorm(modelp, countAB, total); + } + Y_FAIL("no way!"); return 0; } - inline float Score(EEntryScore score, ui32 len, float modelp, ui32 count, ui32 total) { - if (!total) { - return 0; - } - ui32 m = 1; - switch (score) { - case ES_LEN_COUNT: - m = len; + inline float Score(EEntryScore score, ui32 len, float modelp, ui32 count, ui32 total) { + if (!total) { + return 0; + } + ui32 m = 1; + switch (score) { + case ES_LEN_COUNT: + m = len; [[fallthrough]]; - case ES_COUNT: - return m * count; - case ES_LEN_SIMPLE: - m = len; + case ES_COUNT: + return m * count; + case ES_LEN_SIMPLE: + m = len; [[fallthrough]]; - case ES_SIMPLE: - return m * SimpleTest(modelp, count, total); - case ES_SOLAR: - return SolarTest(modelp, count, total); - } - Y_FAIL("no way!"); + case ES_SIMPLE: + return m * SimpleTest(modelp, count, total); + case ES_SOLAR: + return SolarTest(modelp, count, total); + } + Y_FAIL("no way!"); return 0; } diff --git a/library/cpp/codecs/greedy_dict/ut/greedy_dict_ut.cpp b/library/cpp/codecs/greedy_dict/ut/greedy_dict_ut.cpp index 679089a11b..e33976d333 100644 --- a/library/cpp/codecs/greedy_dict/ut/greedy_dict_ut.cpp +++ b/library/cpp/codecs/greedy_dict/ut/greedy_dict_ut.cpp @@ -6,11 +6,11 @@ #include <util/generic/ymath.h> class TGreedyDictTest: public TTestBase { - UNIT_TEST_SUITE(TGreedyDictTest); + UNIT_TEST_SUITE(TGreedyDictTest); UNIT_TEST(TestEntrySet) UNIT_TEST(TestBuilder0) UNIT_TEST(TestBuilder) - UNIT_TEST_SUITE_END(); + UNIT_TEST_SUITE_END(); void TestEntrySet() { using namespace NGreedyDict; @@ -120,7 +120,7 @@ class TGreedyDictTest: public TTestBase { } void FillData(NGreedyDict::TStringBufs& data) { - static const char* urls[] = {"http://53.ru/car/motors/foreign/opel/tigra/", "http://abakan.24au.ru/tender/85904/", "http://anm15.gulaig.com/", "http://avto-parts.com/mercedes-benz/mercedes-benz-w220-1998-2005/category-442/category-443/", "http://ballooncousin.co.uk/", "http://benzol.ru/equipment/?id=1211&parent=514", "http://blazingseorank.com/blazing-seo-rank-free-website-analysis-to-increase-rank-and-traffic-450.html", "http://blogblaugrana.contadorwebmasters.com/", "http://bristolhash.org.uk/bh3cntct.php", "http://broker.borovichi.ru/category/item/3/1/0/8/28/257", "http://canoncompactcamerax.blogspot.com/", "http://classifieds.smashits.com/p,107881,email-to-friend.htm", "http://conferences.ksde.org/Portals/132/FallAssessment/SAVETHEDAY-FA09.pdf", "http://eway.vn/raovat/325-dien-tu-gia-dung/337-dieu-hoa/98041-b1-sua-may-lanh-quan-binh-tan-sua-may-lanh-quan-binh-chanh-hh-979676119-toan-quoc.html", "http://gallery.e2bn.org/asset73204_8-.html", "http://goplay.nsw.gov.au/activities-for-kids/by/historic-houses-trust/?startdate=2012-07-10", "http://grichards19067.multiply.com/", "http://hotkovo.egent.ru/user/89262269084/", "http://howimetyourself.com/?redirect_to=http://gomiso.com/m/suits/seasons/2/episodes/2", "http://islamqa.com/hi/ref/9014/DEAD%20PEOPLE%20GOOD%20DEEDS", "http://lapras.rutube.ru/", "http://nceluiko.ya.ru/", "http://nyanyanyanyaa.beon.ru/", "http://ozbo.com/Leaf-River-DV-7SS-7-0-MP-Game-Camera-K1-32541.html", "http://sbantom.ru/catalog/chasy/632753.html", "http://shopingoff.com/index.php?option=com_virtuemart&Itemid=65&category_id=&page=shop.browse&manufacturer_id=122&limit=32&limitstart=96", "http://shopingoff.com/katalog-odezhdy/manufacturer/62-christian-audigier.html?limit=32&start=448", "https://webwinkel.ah.nl/process?fh_location=//ecommerce/nl_NL/categories%3C%7Becommerce_shoc1%7D/it_show_product_code_1384%3E%7B10%3B20%7D/pr_startdate%3C20120519/pr_enddate%3E20120519/pr_ltc_allowed%3E%7Bbowi%7D/categories%3C%7Becommerce_shoc1_1al%7D/categories%3C%7Becommerce_shoc1_1al_1ahal%7D&&action=albert_noscript.modules.build", "http://top100.rambler.ru/navi/?theme=208/210/371&rgn=17", "http://volgogradskaya-oblast.extra-m.ru/classifieds/rabota/vakansii/banki-investicii/901467/", "http://wikien4.appspot.com/wiki/Warburg_hypothesis", "http://wola_baranowska.kamerzysta24.com.pl/", "http://www.10dot0dot0dot1.com/", "http://www.anima-redux.ru/index.php?key=gifts+teenage+girls", "http://www.aquaticabyseaworld.com/Calendar.aspx/CP/CP/CP/sp-us/CP/CP/ParkMap/Tickets/Weather.aspx", "http://www.autousa.com/360spin/2012_cadillac_ctssportwagon_3.6awdpremiumcollection.htm", "http://www.booking.com/city/gb/paignton-aireborough.html?inac=0&lang=pl", "http://www.booking.com/city/it/vodo-cadore.en.html", "http://www.booking.com/district/us/new-york/rockefeller-center.html&lang=no", "http://www.booking.com/hotel/bg/crown-fort-club.lv.html", "http://www.booking.com/hotel/ca/gouverneur-rimouski.ar.html", "http://www.booking.com/hotel/ch/l-auberge-du-chalet-a-gobet.fi.html", "http://www.booking.com/hotel/de/mark-garni.ru.html?aid=337384;label=yandex-hotel-mark-garni-68157-%7Bparam1%7D", "http://www.booking.com/hotel/de/mercure-goldschmieding-castrop-rauxel.ro.html", "http://www.booking.com/hotel/de/zollenspieker-fahrhaus.fr.html", "http://www.booking.com/hotel/es/jardin-metropolitano.ca.html", "http://www.booking.com/hotel/fr/clim.fr.html", "http://www.booking.com/hotel/fr/radisson-sas-toulouse-airport.et.html", "http://www.booking.com/hotel/gb/stgileshotel.ro.html?srfid=68c7fe42a03653a8796c84435c5299e4X16?tab=4", "http://www.booking.com/hotel/gr/rodos-park-suites.ru.html", "http://www.booking.com/hotel/id/le-grande-suites-bali.ru.html", "http://www.booking.com/hotel/it/mozart.it.html?aid=321655", "http://www.booking.com/hotel/ni/bahia-del-sol-villas.ru.html?dcid=1;dva=0", "http://www.booking.com/hotel/nl/cpschiphol.ro.html.ro.html?tab=4", "http://www.booking.com/hotel/th/laem-din.en-gb.html", "http://www.booking.com/hotel/th/tinidee-ranong.en.html", "http://www.booking.com/hotel/us/best-western-plus-merrimack-valley.hu.html", "http://www.booking.com/hotel/vn/tan-hai-long.km.html", "http://www.booking.com/landmark/au/royal-brisbane-women-s-hospital.vi.html", "http://www.booking.com/landmark/hk/nam-cheong-station.html&lang=id", "http://www.booking.com/landmark/it/spanish-steps.ca.html", "http://www.booking.com/landmark/sg/asian-civilisations-museum.html&lang=fi", "http://www.booking.com/place/fi-1376029.pt.html", "http://www.booking.com/place/tn257337.pl.html", "http://www.booking.com/region/ca/niagarafalls.ar.html&selected_currency=PLN", "http://www.booking.com/region/mx/queretaro.pt-pt.html&selected_currency=AUD", "http://www.booking.com/searchresults.en.html?city=20063074", "http://www.booking.com/searchresults.et.html?checkin=;checkout=;city=-394632", "http://www.booking.com/searchresults.lv.html?region=3936", "http://www.cevredanismanlari.com/index.php/component/k2/index.php/mevzuat/genel-yazlar/item/dosyalar/index.php?option=com_k2&view=item&id=16:iso-14001-%C3%A7evre-y%C3%B6netim-sistemi&Itemid=132&limitstart=107120", "http://www.dh-wholesaler.com/MENS-POLO-RACING-TEE-RL-p-417.html", "http://www.employabilityonline.net/", "http://www.esso.inc.ru/board/tools.php?event=profile&pname=Invinerrq", "http://www.filesurgery.ru/searchfw/kids_clothes-3.html", "http://www.furnitureandcarpetsource.com/Item.aspx?ItemID=-2107311899&ItemNum=53-T3048", "http://www.gets.cn/product/Gold-Sand-Lampwork-Glass-Beads--Flat-round--28x28x13mm_p260717.html", "http://www.gets.cn/wholesale-Sterling-Silver-Pendant-Findings-3577_S--L-Star-P-1.html?view=1&by=1", "http://www.homeandgardenadvice.com/diy/Mortgages_Loans_and_Financing/9221.html", "http://www.hongkongairport.com/eng/index.html/passenger/passenger/transport/to-from-airport/business/about-the-airport/transport/shopping/entertainment/t2/passenger/interactive-map.html", "http://www.hongkongairport.com/eng/index.html/shopping/insideshopping/all/passenger/transfer-transit/all/airline-information/shopping/entertainment/t2/business/about-the-airport/welcome.html", "http://www.hongkongairport.com/eng/index.html/transport/business/about-the-airport/transport/business/airport-authority/passenger/shopping/dining/all/dining.html", "http://www.idedge.com/index.cfm/fuseaction/category.display/category_id/298/index.cfm", "http://www.istanbulburda.com/aramalar.php", "http://www.jewelryinthenet.com/ads/AdDetail.aspx?AdID=1-0311002490689&stid=22-0111001020877", "http://www.johnnydepp.ru/forum/index.php?showtopic=1629&mode=linearplus&view=findpost&p=186977", "http://www.johnnydepp.ru/forum/index.php?showtopic=476&st=60&p=87379&", "http://www.joseleano.com/joomla/index.php/audio", "http://www.kaplicarehberi.com/tag/sakar-ilicali-kaplicalari/feed", "http://www.khaber.com.tr/arama.html?key=%C3%A7avdar", "http://www.kiz-oyunlari1.com/1783/4437/4363/1056/4170/Bump-Copter2-.html", "http://www.kiz-oyunlari1.com/3752/2612/4175/1166/3649/1047/Angelina-Oyunu.html", "http://www.kiz-oyunlari1.com/4266/3630/3665/3286/4121/301/3274/Sinir-Sinekler-.html", "http://www.kuldiga.lv/index.php?f=8&cat=371", "http://www.kuldiga.lv/index.php/img/index.php?l=lv&art_id=1836&show_c=&cat=85", "http://www.patronessa.ru/remontiruemsya/kuzovnie30raboti.html", "http://www.rapdict.org/Nu_Money?title=Talk:Nu_Money&action=edit", "http://www.serafin-phu.tabor24.com/?page=8", "http://www.shoes-store.org/brand1/Kids/Minnetonka.html", "http://www.shoes-store.org/shoes-store.xml", "http://www.way2allah.com/khotab-download-34695.htm"}; + static const char* urls[] = {"http://53.ru/car/motors/foreign/opel/tigra/", "http://abakan.24au.ru/tender/85904/", "http://anm15.gulaig.com/", "http://avto-parts.com/mercedes-benz/mercedes-benz-w220-1998-2005/category-442/category-443/", "http://ballooncousin.co.uk/", "http://benzol.ru/equipment/?id=1211&parent=514", "http://blazingseorank.com/blazing-seo-rank-free-website-analysis-to-increase-rank-and-traffic-450.html", "http://blogblaugrana.contadorwebmasters.com/", "http://bristolhash.org.uk/bh3cntct.php", "http://broker.borovichi.ru/category/item/3/1/0/8/28/257", "http://canoncompactcamerax.blogspot.com/", "http://classifieds.smashits.com/p,107881,email-to-friend.htm", "http://conferences.ksde.org/Portals/132/FallAssessment/SAVETHEDAY-FA09.pdf", "http://eway.vn/raovat/325-dien-tu-gia-dung/337-dieu-hoa/98041-b1-sua-may-lanh-quan-binh-tan-sua-may-lanh-quan-binh-chanh-hh-979676119-toan-quoc.html", "http://gallery.e2bn.org/asset73204_8-.html", "http://goplay.nsw.gov.au/activities-for-kids/by/historic-houses-trust/?startdate=2012-07-10", "http://grichards19067.multiply.com/", "http://hotkovo.egent.ru/user/89262269084/", "http://howimetyourself.com/?redirect_to=http://gomiso.com/m/suits/seasons/2/episodes/2", "http://islamqa.com/hi/ref/9014/DEAD%20PEOPLE%20GOOD%20DEEDS", "http://lapras.rutube.ru/", "http://nceluiko.ya.ru/", "http://nyanyanyanyaa.beon.ru/", "http://ozbo.com/Leaf-River-DV-7SS-7-0-MP-Game-Camera-K1-32541.html", "http://sbantom.ru/catalog/chasy/632753.html", "http://shopingoff.com/index.php?option=com_virtuemart&Itemid=65&category_id=&page=shop.browse&manufacturer_id=122&limit=32&limitstart=96", "http://shopingoff.com/katalog-odezhdy/manufacturer/62-christian-audigier.html?limit=32&start=448", "https://webwinkel.ah.nl/process?fh_location=//ecommerce/nl_NL/categories%3C%7Becommerce_shoc1%7D/it_show_product_code_1384%3E%7B10%3B20%7D/pr_startdate%3C20120519/pr_enddate%3E20120519/pr_ltc_allowed%3E%7Bbowi%7D/categories%3C%7Becommerce_shoc1_1al%7D/categories%3C%7Becommerce_shoc1_1al_1ahal%7D&&action=albert_noscript.modules.build", "http://top100.rambler.ru/navi/?theme=208/210/371&rgn=17", "http://volgogradskaya-oblast.extra-m.ru/classifieds/rabota/vakansii/banki-investicii/901467/", "http://wikien4.appspot.com/wiki/Warburg_hypothesis", "http://wola_baranowska.kamerzysta24.com.pl/", "http://www.10dot0dot0dot1.com/", "http://www.anima-redux.ru/index.php?key=gifts+teenage+girls", "http://www.aquaticabyseaworld.com/Calendar.aspx/CP/CP/CP/sp-us/CP/CP/ParkMap/Tickets/Weather.aspx", "http://www.autousa.com/360spin/2012_cadillac_ctssportwagon_3.6awdpremiumcollection.htm", "http://www.booking.com/city/gb/paignton-aireborough.html?inac=0&lang=pl", "http://www.booking.com/city/it/vodo-cadore.en.html", "http://www.booking.com/district/us/new-york/rockefeller-center.html&lang=no", "http://www.booking.com/hotel/bg/crown-fort-club.lv.html", "http://www.booking.com/hotel/ca/gouverneur-rimouski.ar.html", "http://www.booking.com/hotel/ch/l-auberge-du-chalet-a-gobet.fi.html", "http://www.booking.com/hotel/de/mark-garni.ru.html?aid=337384;label=yandex-hotel-mark-garni-68157-%7Bparam1%7D", "http://www.booking.com/hotel/de/mercure-goldschmieding-castrop-rauxel.ro.html", "http://www.booking.com/hotel/de/zollenspieker-fahrhaus.fr.html", "http://www.booking.com/hotel/es/jardin-metropolitano.ca.html", "http://www.booking.com/hotel/fr/clim.fr.html", "http://www.booking.com/hotel/fr/radisson-sas-toulouse-airport.et.html", "http://www.booking.com/hotel/gb/stgileshotel.ro.html?srfid=68c7fe42a03653a8796c84435c5299e4X16?tab=4", "http://www.booking.com/hotel/gr/rodos-park-suites.ru.html", "http://www.booking.com/hotel/id/le-grande-suites-bali.ru.html", "http://www.booking.com/hotel/it/mozart.it.html?aid=321655", "http://www.booking.com/hotel/ni/bahia-del-sol-villas.ru.html?dcid=1;dva=0", "http://www.booking.com/hotel/nl/cpschiphol.ro.html.ro.html?tab=4", "http://www.booking.com/hotel/th/laem-din.en-gb.html", "http://www.booking.com/hotel/th/tinidee-ranong.en.html", "http://www.booking.com/hotel/us/best-western-plus-merrimack-valley.hu.html", "http://www.booking.com/hotel/vn/tan-hai-long.km.html", "http://www.booking.com/landmark/au/royal-brisbane-women-s-hospital.vi.html", "http://www.booking.com/landmark/hk/nam-cheong-station.html&lang=id", "http://www.booking.com/landmark/it/spanish-steps.ca.html", "http://www.booking.com/landmark/sg/asian-civilisations-museum.html&lang=fi", "http://www.booking.com/place/fi-1376029.pt.html", "http://www.booking.com/place/tn257337.pl.html", "http://www.booking.com/region/ca/niagarafalls.ar.html&selected_currency=PLN", "http://www.booking.com/region/mx/queretaro.pt-pt.html&selected_currency=AUD", "http://www.booking.com/searchresults.en.html?city=20063074", "http://www.booking.com/searchresults.et.html?checkin=;checkout=;city=-394632", "http://www.booking.com/searchresults.lv.html?region=3936", "http://www.cevredanismanlari.com/index.php/component/k2/index.php/mevzuat/genel-yazlar/item/dosyalar/index.php?option=com_k2&view=item&id=16:iso-14001-%C3%A7evre-y%C3%B6netim-sistemi&Itemid=132&limitstart=107120", "http://www.dh-wholesaler.com/MENS-POLO-RACING-TEE-RL-p-417.html", "http://www.employabilityonline.net/", "http://www.esso.inc.ru/board/tools.php?event=profile&pname=Invinerrq", "http://www.filesurgery.ru/searchfw/kids_clothes-3.html", "http://www.furnitureandcarpetsource.com/Item.aspx?ItemID=-2107311899&ItemNum=53-T3048", "http://www.gets.cn/product/Gold-Sand-Lampwork-Glass-Beads--Flat-round--28x28x13mm_p260717.html", "http://www.gets.cn/wholesale-Sterling-Silver-Pendant-Findings-3577_S--L-Star-P-1.html?view=1&by=1", "http://www.homeandgardenadvice.com/diy/Mortgages_Loans_and_Financing/9221.html", "http://www.hongkongairport.com/eng/index.html/passenger/passenger/transport/to-from-airport/business/about-the-airport/transport/shopping/entertainment/t2/passenger/interactive-map.html", "http://www.hongkongairport.com/eng/index.html/shopping/insideshopping/all/passenger/transfer-transit/all/airline-information/shopping/entertainment/t2/business/about-the-airport/welcome.html", "http://www.hongkongairport.com/eng/index.html/transport/business/about-the-airport/transport/business/airport-authority/passenger/shopping/dining/all/dining.html", "http://www.idedge.com/index.cfm/fuseaction/category.display/category_id/298/index.cfm", "http://www.istanbulburda.com/aramalar.php", "http://www.jewelryinthenet.com/ads/AdDetail.aspx?AdID=1-0311002490689&stid=22-0111001020877", "http://www.johnnydepp.ru/forum/index.php?showtopic=1629&mode=linearplus&view=findpost&p=186977", "http://www.johnnydepp.ru/forum/index.php?showtopic=476&st=60&p=87379&", "http://www.joseleano.com/joomla/index.php/audio", "http://www.kaplicarehberi.com/tag/sakar-ilicali-kaplicalari/feed", "http://www.khaber.com.tr/arama.html?key=%C3%A7avdar", "http://www.kiz-oyunlari1.com/1783/4437/4363/1056/4170/Bump-Copter2-.html", "http://www.kiz-oyunlari1.com/3752/2612/4175/1166/3649/1047/Angelina-Oyunu.html", "http://www.kiz-oyunlari1.com/4266/3630/3665/3286/4121/301/3274/Sinir-Sinekler-.html", "http://www.kuldiga.lv/index.php?f=8&cat=371", "http://www.kuldiga.lv/index.php/img/index.php?l=lv&art_id=1836&show_c=&cat=85", "http://www.patronessa.ru/remontiruemsya/kuzovnie30raboti.html", "http://www.rapdict.org/Nu_Money?title=Talk:Nu_Money&action=edit", "http://www.serafin-phu.tabor24.com/?page=8", "http://www.shoes-store.org/brand1/Kids/Minnetonka.html", "http://www.shoes-store.org/shoes-store.xml", "http://www.way2allah.com/khotab-download-34695.htm"}; data.clear(); data.insert(data.begin(), urls, urls + Y_ARRAY_SIZE(urls)); } @@ -128,7 +128,7 @@ class TGreedyDictTest: public TTestBase { typedef THashMap<TStringBuf, NGreedyDict::TEntry> TDict; TAutoPtr<NGreedyDict::TEntrySet> DoTestBuilder(const NGreedyDict::TBuildSettings& s, - TDict& res) { + TDict& res) { using namespace NGreedyDict; TStringBufs data; diff --git a/library/cpp/codecs/huffman_codec.cpp b/library/cpp/codecs/huffman_codec.cpp index 650fe7cdfd..c8b126ccd0 100644 --- a/library/cpp/codecs/huffman_codec.cpp +++ b/library/cpp/codecs/huffman_codec.cpp @@ -9,584 +9,584 @@ #include <util/string/printf.h> namespace NCodecs { - template <typename T> - struct TCanonicalCmp { - bool operator()(const T& a, const T& b) const { - if (a.CodeLength == b.CodeLength) { - return a.Char < b.Char; - } else { - return a.CodeLength < b.CodeLength; - } - } - }; - - template <typename T> - struct TByCharCmp { - bool operator()(const T& a, const T& b) const { + template <typename T> + struct TCanonicalCmp { + bool operator()(const T& a, const T& b) const { + if (a.CodeLength == b.CodeLength) { + return a.Char < b.Char; + } else { + return a.CodeLength < b.CodeLength; + } + } + }; + + template <typename T> + struct TByCharCmp { + bool operator()(const T& a, const T& b) const { return a.Char < b.Char; } - }; + }; - struct TTreeEntry { - static const ui32 InvalidBranch = (ui32)-1; + struct TTreeEntry { + static const ui32 InvalidBranch = (ui32)-1; - ui64 Freq = 0; - ui32 Branches[2]{InvalidBranch, InvalidBranch}; + ui64 Freq = 0; + ui32 Branches[2]{InvalidBranch, InvalidBranch}; - ui32 CodeLength = 0; - ui8 Char = 0; - bool Invalid = false; + ui32 CodeLength = 0; + ui8 Char = 0; + bool Invalid = false; - TTreeEntry() = default; + TTreeEntry() = default; - static bool ByFreq(const TTreeEntry& a, const TTreeEntry& b) { - return a.Freq < b.Freq; - } + static bool ByFreq(const TTreeEntry& a, const TTreeEntry& b) { + return a.Freq < b.Freq; + } - static bool ByFreqRev(const TTreeEntry& a, const TTreeEntry& b) { - return a.Freq > b.Freq; - } - }; + static bool ByFreqRev(const TTreeEntry& a, const TTreeEntry& b) { + return a.Freq > b.Freq; + } + }; - using TCodeTree = TVector<TTreeEntry>; + using TCodeTree = TVector<TTreeEntry>; - void InitTreeByFreqs(TCodeTree& tree, const ui64 freqs[256]) { - tree.reserve(255 * 256 / 2); // worst case - balanced tree + void InitTreeByFreqs(TCodeTree& tree, const ui64 freqs[256]) { + tree.reserve(255 * 256 / 2); // worst case - balanced tree - for (ui32 i = 0; i < 256; ++i) { - tree.emplace_back(); - tree.back().Char = i; - tree.back().Freq = freqs[i]; - } + for (ui32 i = 0; i < 256; ++i) { + tree.emplace_back(); + tree.back().Char = i; + tree.back().Freq = freqs[i]; + } - StableSort(tree.begin(), tree.end(), TTreeEntry::ByFreq); + StableSort(tree.begin(), tree.end(), TTreeEntry::ByFreq); } - void InitTree(TCodeTree& tree, ISequenceReader* in) { - using namespace NPrivate; - ui64 freqs[256]; - Zero(freqs); + void InitTree(TCodeTree& tree, ISequenceReader* in) { + using namespace NPrivate; + ui64 freqs[256]; + Zero(freqs); - TStringBuf r; - while (in->NextRegion(r)) { + TStringBuf r; + while (in->NextRegion(r)) { for (ui64 i = 0; i < r.size(); ++i) - ++freqs[(ui8)r[i]]; - } + ++freqs[(ui8)r[i]]; + } - InitTreeByFreqs(tree, freqs); + InitTreeByFreqs(tree, freqs); } - void CalculateCodeLengths(TCodeTree& tree) { - Y_ENSURE(tree.size() == 256, " "); - const ui32 firstbranch = tree.size(); + void CalculateCodeLengths(TCodeTree& tree) { + Y_ENSURE(tree.size() == 256, " "); + const ui32 firstbranch = tree.size(); - ui32 curleaf = 0; - ui32 curbranch = firstbranch; + ui32 curleaf = 0; + ui32 curbranch = firstbranch; - // building code tree. two priority queues are combined in one. - while (firstbranch - curleaf + tree.size() - curbranch >= 2) { - TTreeEntry e; + // building code tree. two priority queues are combined in one. + while (firstbranch - curleaf + tree.size() - curbranch >= 2) { + TTreeEntry e; - for (auto& branche : e.Branches) { - ui32 br; + for (auto& branche : e.Branches) { + ui32 br; - if (curleaf >= firstbranch) - br = curbranch++; - else if (curbranch >= tree.size()) - br = curleaf++; - else if (tree[curleaf].Freq < tree[curbranch].Freq) - br = curleaf++; - else - br = curbranch++; + if (curleaf >= firstbranch) + br = curbranch++; + else if (curbranch >= tree.size()) + br = curleaf++; + else if (tree[curleaf].Freq < tree[curbranch].Freq) + br = curleaf++; + else + br = curbranch++; - Y_ENSURE(br < tree.size(), " "); - branche = br; - e.Freq += tree[br].Freq; - } + Y_ENSURE(br < tree.size(), " "); + branche = br; + e.Freq += tree[br].Freq; + } - tree.push_back(e); - PushHeap(tree.begin() + curbranch, tree.end(), TTreeEntry::ByFreqRev); + tree.push_back(e); + PushHeap(tree.begin() + curbranch, tree.end(), TTreeEntry::ByFreqRev); } - // computing code lengths - for (ui64 i = tree.size() - 1; i >= firstbranch; --i) { - TTreeEntry e = tree[i]; + // computing code lengths + for (ui64 i = tree.size() - 1; i >= firstbranch; --i) { + TTreeEntry e = tree[i]; - for (auto branche : e.Branches) - tree[branche].CodeLength = e.CodeLength + 1; - } - - // chopping off the branches - tree.resize(firstbranch); + for (auto branche : e.Branches) + tree[branche].CodeLength = e.CodeLength + 1; + } - Sort(tree.begin(), tree.end(), TCanonicalCmp<TTreeEntry>()); + // chopping off the branches + tree.resize(firstbranch); - // simplification: we are stripping codes longer than 64 bits - while (!tree.empty() && tree.back().CodeLength > 64) - tree.pop_back(); + Sort(tree.begin(), tree.end(), TCanonicalCmp<TTreeEntry>()); - // will not compress - if (tree.empty()) - return; + // simplification: we are stripping codes longer than 64 bits + while (!tree.empty() && tree.back().CodeLength > 64) + tree.pop_back(); - // special invalid code word - tree.back().Invalid = true; - } + // will not compress + if (tree.empty()) + return; - struct TEncoderEntry { - ui64 Code = 0; + // special invalid code word + tree.back().Invalid = true; + } - ui8 CodeLength = 0; - ui8 Char = 0; - ui8 Invalid = true; + struct TEncoderEntry { + ui64 Code = 0; - explicit TEncoderEntry(TTreeEntry e) - : CodeLength(e.CodeLength) - , Char(e.Char) - , Invalid(e.Invalid) - { - } + ui8 CodeLength = 0; + ui8 Char = 0; + ui8 Invalid = true; - TEncoderEntry() = default; - }; + explicit TEncoderEntry(TTreeEntry e) + : CodeLength(e.CodeLength) + , Char(e.Char) + , Invalid(e.Invalid) + { + } - struct TEncoderTable { - TEncoderEntry Entries[256]; + TEncoderEntry() = default; + }; - void Save(IOutputStream* out) const { - ui16 nval = 0; + struct TEncoderTable { + TEncoderEntry Entries[256]; - for (auto entrie : Entries) - nval += !entrie.Invalid; - - ::Save(out, nval); - - for (auto entrie : Entries) { - if (!entrie.Invalid) { - ::Save(out, entrie.Char); - ::Save(out, entrie.CodeLength); - } - } - } + void Save(IOutputStream* out) const { + ui16 nval = 0; - void Load(IInputStream* in) { - ui16 nval = 0; - ::Load(in, nval); + for (auto entrie : Entries) + nval += !entrie.Invalid; - for (ui32 i = 0; i < 256; ++i) - Entries[i].Char = i; + ::Save(out, nval); - for (ui32 i = 0; i < nval; ++i) { - ui8 ch = 0; - ui8 len = 0; - ::Load(in, ch); - ::Load(in, len); - Entries[ch].CodeLength = len; - Entries[ch].Invalid = false; + for (auto entrie : Entries) { + if (!entrie.Invalid) { + ::Save(out, entrie.Char); + ::Save(out, entrie.CodeLength); + } } } - }; - - struct TDecoderEntry { - ui32 NextTable : 10; - ui32 Char : 8; - ui32 Invalid : 1; - ui32 Bad : 1; - - TDecoderEntry() - : NextTable() - , Char() - , Invalid() - , Bad() - { - } - }; - - struct TDecoderTable: public TIntrusiveListItem<TDecoderTable> { - ui64 Length = 0; - ui64 BaseCode = 0; - TDecoderEntry Entries[256]; - - TDecoderTable() { - Zero(Entries); + void Load(IInputStream* in) { + ui16 nval = 0; + ::Load(in, nval); + + for (ui32 i = 0; i < 256; ++i) + Entries[i].Char = i; + + for (ui32 i = 0; i < nval; ++i) { + ui8 ch = 0; + ui8 len = 0; + ::Load(in, ch); + ::Load(in, len); + Entries[ch].CodeLength = len; + Entries[ch].Invalid = false; + } } - }; - - const int CACHE_BITS_COUNT = 16; - class THuffmanCodec::TImpl: public TAtomicRefCount<TImpl> { - TEncoderTable Encoder; - TDecoderTable Decoder[256]; - - TEncoderEntry Invalid; - - ui32 SubTablesNum; - - class THuffmanCache { - struct TCacheEntry { - int EndOffset : 24; - int BitsLeft : 8; - }; - TVector<char> DecodeCache; - TVector<TCacheEntry> CacheEntries; - const TImpl& Original; - - public: - THuffmanCache(const THuffmanCodec::TImpl& encoder); - - void Decode(NBitIO::TBitInput& in, TBuffer& out) const; + }; + + struct TDecoderEntry { + ui32 NextTable : 10; + ui32 Char : 8; + ui32 Invalid : 1; + ui32 Bad : 1; + + TDecoderEntry() + : NextTable() + , Char() + , Invalid() + , Bad() + { + } + }; + + struct TDecoderTable: public TIntrusiveListItem<TDecoderTable> { + ui64 Length = 0; + ui64 BaseCode = 0; + + TDecoderEntry Entries[256]; + + TDecoderTable() { + Zero(Entries); + } + }; + + const int CACHE_BITS_COUNT = 16; + class THuffmanCodec::TImpl: public TAtomicRefCount<TImpl> { + TEncoderTable Encoder; + TDecoderTable Decoder[256]; + + TEncoderEntry Invalid; + + ui32 SubTablesNum; + + class THuffmanCache { + struct TCacheEntry { + int EndOffset : 24; + int BitsLeft : 8; + }; + TVector<char> DecodeCache; + TVector<TCacheEntry> CacheEntries; + const TImpl& Original; + + public: + THuffmanCache(const THuffmanCodec::TImpl& encoder); + + void Decode(NBitIO::TBitInput& in, TBuffer& out) const; }; - THolder<THuffmanCache> Cache; + THolder<THuffmanCache> Cache; - public: - TImpl() - : SubTablesNum(1) - { - Invalid.CodeLength = 255; - } + public: + TImpl() + : SubTablesNum(1) + { + Invalid.CodeLength = 255; + } - ui8 Encode(TStringBuf in, TBuffer& out) const { - out.Clear(); + ui8 Encode(TStringBuf in, TBuffer& out) const { + out.Clear(); if (in.empty()) { - return 0; - } + return 0; + } out.Reserve(in.size() * 2); - { - NBitIO::TBitOutputVector<TBuffer> bout(&out); - TStringBuf tin = in; + { + NBitIO::TBitOutputVector<TBuffer> bout(&out); + TStringBuf tin = in; - // data is under compression - bout.Write(1, 1); + // data is under compression + bout.Write(1, 1); - for (auto t : tin) { - const TEncoderEntry& ce = Encoder.Entries[(ui8)t]; + for (auto t : tin) { + const TEncoderEntry& ce = Encoder.Entries[(ui8)t]; - bout.Write(ce.Code, ce.CodeLength); + bout.Write(ce.Code, ce.CodeLength); - if (ce.Invalid) { - bout.Write(t, 8); - } - } + if (ce.Invalid) { + bout.Write(t, 8); + } + } - // in canonical huffman coding there cannot be a code having no 0 in the suffix - // and shorter than 8 bits. - bout.Write((ui64)-1, bout.GetByteReminder()); - return bout.GetByteReminder(); + // in canonical huffman coding there cannot be a code having no 0 in the suffix + // and shorter than 8 bits. + bout.Write((ui64)-1, bout.GetByteReminder()); + return bout.GetByteReminder(); } } - void Decode(TStringBuf in, TBuffer& out) const { - out.Clear(); + void Decode(TStringBuf in, TBuffer& out) const { + out.Clear(); if (in.empty()) { - return; - } + return; + } - NBitIO::TBitInput bin(in); - ui64 f = 0; - bin.ReadK<1>(f); + NBitIO::TBitInput bin(in); + ui64 f = 0; + bin.ReadK<1>(f); - // if data is uncompressed - if (!f) { - in.Skip(1); + // if data is uncompressed + if (!f) { + in.Skip(1); out.Append(in.data(), in.size()); - } else { + } else { out.Reserve(in.size() * 8); - if (Cache.Get()) { - Cache->Decode(bin, out); - } else { - while (ReadNextChar(bin, out)) { - } + if (Cache.Get()) { + Cache->Decode(bin, out); + } else { + while (ReadNextChar(bin, out)) { + } } } } - Y_FORCE_INLINE int ReadNextChar(NBitIO::TBitInput& bin, TBuffer& out) const { - const TDecoderTable* table = Decoder; - TDecoderEntry e; + Y_FORCE_INLINE int ReadNextChar(NBitIO::TBitInput& bin, TBuffer& out) const { + const TDecoderTable* table = Decoder; + TDecoderEntry e; - int bitsRead = 0; - while (true) { - ui64 code = 0; + int bitsRead = 0; + while (true) { + ui64 code = 0; - if (Y_UNLIKELY(!bin.Read(code, table->Length))) - return 0; - bitsRead += table->Length; + if (Y_UNLIKELY(!bin.Read(code, table->Length))) + return 0; + bitsRead += table->Length; - if (Y_UNLIKELY(code < table->BaseCode)) - return 0; + if (Y_UNLIKELY(code < table->BaseCode)) + return 0; - code -= table->BaseCode; + code -= table->BaseCode; - if (Y_UNLIKELY(code > 255)) - return 0; + if (Y_UNLIKELY(code > 255)) + return 0; - e = table->Entries[code]; + e = table->Entries[code]; - if (Y_UNLIKELY(e.Bad)) - return 0; + if (Y_UNLIKELY(e.Bad)) + return 0; - if (e.NextTable) { - table = Decoder + e.NextTable; + if (e.NextTable) { + table = Decoder + e.NextTable; } else { - if (e.Invalid) { - code = 0; - bin.ReadK<8>(code); - bitsRead += 8; - out.Append((ui8)code); - } else { - out.Append((ui8)e.Char); - } - - return bitsRead; + if (e.Invalid) { + code = 0; + bin.ReadK<8>(code); + bitsRead += 8; + out.Append((ui8)code); + } else { + out.Append((ui8)e.Char); + } + + return bitsRead; } - } + } - Y_ENSURE(false, " could not decode input"); - return 0; + Y_ENSURE(false, " could not decode input"); + return 0; } - void GenerateEncoder(TCodeTree& tree) { - const ui64 sz = tree.size(); + void GenerateEncoder(TCodeTree& tree) { + const ui64 sz = tree.size(); - TEncoderEntry lastcode = Encoder.Entries[tree[0].Char] = TEncoderEntry(tree[0]); + TEncoderEntry lastcode = Encoder.Entries[tree[0].Char] = TEncoderEntry(tree[0]); - for (ui32 i = 1; i < sz; ++i) { - const TTreeEntry& te = tree[i]; - TEncoderEntry& e = Encoder.Entries[te.Char]; - e = TEncoderEntry(te); + for (ui32 i = 1; i < sz; ++i) { + const TTreeEntry& te = tree[i]; + TEncoderEntry& e = Encoder.Entries[te.Char]; + e = TEncoderEntry(te); - e.Code = (lastcode.Code + 1) << (e.CodeLength - lastcode.CodeLength); - lastcode = e; + e.Code = (lastcode.Code + 1) << (e.CodeLength - lastcode.CodeLength); + lastcode = e; - e.Code = ReverseBits(e.Code, e.CodeLength); - - if (e.Invalid) - Invalid = e; - } + e.Code = ReverseBits(e.Code, e.CodeLength); - for (auto& e : Encoder.Entries) { - if (e.Invalid) - e = Invalid; + if (e.Invalid) + Invalid = e; + } - Y_ENSURE(e.CodeLength, " "); - } + for (auto& e : Encoder.Entries) { + if (e.Invalid) + e = Invalid; + + Y_ENSURE(e.CodeLength, " "); + } } - void RegenerateEncoder() { - for (auto& entrie : Encoder.Entries) { - if (entrie.Invalid) - entrie.CodeLength = Invalid.CodeLength; - } + void RegenerateEncoder() { + for (auto& entrie : Encoder.Entries) { + if (entrie.Invalid) + entrie.CodeLength = Invalid.CodeLength; + } - Sort(Encoder.Entries, Encoder.Entries + 256, TCanonicalCmp<TEncoderEntry>()); + Sort(Encoder.Entries, Encoder.Entries + 256, TCanonicalCmp<TEncoderEntry>()); - TEncoderEntry lastcode = Encoder.Entries[0]; + TEncoderEntry lastcode = Encoder.Entries[0]; - for (ui32 i = 1; i < 256; ++i) { - TEncoderEntry& e = Encoder.Entries[i]; - e.Code = (lastcode.Code + 1) << (e.CodeLength - lastcode.CodeLength); - lastcode = e; + for (ui32 i = 1; i < 256; ++i) { + TEncoderEntry& e = Encoder.Entries[i]; + e.Code = (lastcode.Code + 1) << (e.CodeLength - lastcode.CodeLength); + lastcode = e; - e.Code = ReverseBits(e.Code, e.CodeLength); - } + e.Code = ReverseBits(e.Code, e.CodeLength); + } - for (auto& entrie : Encoder.Entries) { - if (entrie.Invalid) { - Invalid = entrie; - break; - } - } + for (auto& entrie : Encoder.Entries) { + if (entrie.Invalid) { + Invalid = entrie; + break; + } + } - Sort(Encoder.Entries, Encoder.Entries + 256, TByCharCmp<TEncoderEntry>()); + Sort(Encoder.Entries, Encoder.Entries + 256, TByCharCmp<TEncoderEntry>()); - for (auto& entrie : Encoder.Entries) { - if (entrie.Invalid) - entrie = Invalid; + for (auto& entrie : Encoder.Entries) { + if (entrie.Invalid) + entrie = Invalid; } } - void BuildDecoder() { - TEncoderTable enc = Encoder; - Sort(enc.Entries, enc.Entries + 256, TCanonicalCmp<TEncoderEntry>()); - - TEncoderEntry& e1 = enc.Entries[0]; - Decoder[0].BaseCode = e1.Code; - Decoder[0].Length = e1.CodeLength; - - for (auto e2 : enc.Entries) { - SetEntry(Decoder, e2.Code, e2.CodeLength, e2); - } - Cache.Reset(new THuffmanCache(*this)); + void BuildDecoder() { + TEncoderTable enc = Encoder; + Sort(enc.Entries, enc.Entries + 256, TCanonicalCmp<TEncoderEntry>()); + + TEncoderEntry& e1 = enc.Entries[0]; + Decoder[0].BaseCode = e1.Code; + Decoder[0].Length = e1.CodeLength; + + for (auto e2 : enc.Entries) { + SetEntry(Decoder, e2.Code, e2.CodeLength, e2); + } + Cache.Reset(new THuffmanCache(*this)); } - void SetEntry(TDecoderTable* t, ui64 code, ui64 len, TEncoderEntry e) { - Y_ENSURE(len >= t->Length, len << " < " << t->Length); + void SetEntry(TDecoderTable* t, ui64 code, ui64 len, TEncoderEntry e) { + Y_ENSURE(len >= t->Length, len << " < " << t->Length); - ui64 idx = (code & MaskLowerBits(t->Length)) - t->BaseCode; - TDecoderEntry& d = t->Entries[idx]; + ui64 idx = (code & MaskLowerBits(t->Length)) - t->BaseCode; + TDecoderEntry& d = t->Entries[idx]; - if (len == t->Length) { - Y_ENSURE(!d.NextTable, " "); + if (len == t->Length) { + Y_ENSURE(!d.NextTable, " "); - d.Char = e.Char; - d.Invalid = e.Invalid; - return; - } + d.Char = e.Char; + d.Invalid = e.Invalid; + return; + } - if (!d.NextTable) { - Y_ENSURE(SubTablesNum < Y_ARRAY_SIZE(Decoder), " "); - d.NextTable = SubTablesNum++; - TDecoderTable* nt = Decoder + d.NextTable; - nt->Length = Min<ui64>(8, len - t->Length); - nt->BaseCode = (code >> t->Length) & MaskLowerBits(nt->Length); - } + if (!d.NextTable) { + Y_ENSURE(SubTablesNum < Y_ARRAY_SIZE(Decoder), " "); + d.NextTable = SubTablesNum++; + TDecoderTable* nt = Decoder + d.NextTable; + nt->Length = Min<ui64>(8, len - t->Length); + nt->BaseCode = (code >> t->Length) & MaskLowerBits(nt->Length); + } - SetEntry(Decoder + d.NextTable, code >> t->Length, len - t->Length, e); + SetEntry(Decoder + d.NextTable, code >> t->Length, len - t->Length, e); } - void Learn(ISequenceReader* in) { - { - TCodeTree tree; - InitTree(tree, in); - CalculateCodeLengths(tree); - Y_ENSURE(!tree.empty(), " "); - GenerateEncoder(tree); - } - BuildDecoder(); + void Learn(ISequenceReader* in) { + { + TCodeTree tree; + InitTree(tree, in); + CalculateCodeLengths(tree); + Y_ENSURE(!tree.empty(), " "); + GenerateEncoder(tree); + } + BuildDecoder(); } void LearnByFreqs(const TArrayRef<std::pair<char, ui64>>& freqs) { TCodeTree tree; - ui64 freqsArray[256]; - Zero(freqsArray); + ui64 freqsArray[256]; + Zero(freqsArray); - for (const auto& freq : freqs) - freqsArray[static_cast<ui8>(freq.first)] += freq.second; + for (const auto& freq : freqs) + freqsArray[static_cast<ui8>(freq.first)] += freq.second; - InitTreeByFreqs(tree, freqsArray); - CalculateCodeLengths(tree); + InitTreeByFreqs(tree, freqsArray); + CalculateCodeLengths(tree); - Y_ENSURE(!tree.empty(), " "); + Y_ENSURE(!tree.empty(), " "); - GenerateEncoder(tree); - BuildDecoder(); - } + GenerateEncoder(tree); + BuildDecoder(); + } - void Save(IOutputStream* out) { - ::Save(out, Invalid.CodeLength); - Encoder.Save(out); - } + void Save(IOutputStream* out) { + ::Save(out, Invalid.CodeLength); + Encoder.Save(out); + } - void Load(IInputStream* in) { - ::Load(in, Invalid.CodeLength); - Encoder.Load(in); - RegenerateEncoder(); - BuildDecoder(); - } - }; + void Load(IInputStream* in) { + ::Load(in, Invalid.CodeLength); + Encoder.Load(in); + RegenerateEncoder(); + BuildDecoder(); + } + }; - THuffmanCodec::TImpl::THuffmanCache::THuffmanCache(const THuffmanCodec::TImpl& codec) - : Original(codec) - { - CacheEntries.resize(1 << CACHE_BITS_COUNT); + THuffmanCodec::TImpl::THuffmanCache::THuffmanCache(const THuffmanCodec::TImpl& codec) + : Original(codec) + { + CacheEntries.resize(1 << CACHE_BITS_COUNT); DecodeCache.reserve(CacheEntries.size() * 2); - char buffer[2]; - TBuffer decoded; + char buffer[2]; + TBuffer decoded; for (size_t i = 0; i < CacheEntries.size(); i++) { - buffer[1] = i >> 8; - buffer[0] = i; - NBitIO::TBitInput bin(buffer, buffer + sizeof(buffer)); - int totalBits = 0; - while (true) { - decoded.Resize(0); - int bits = codec.ReadNextChar(bin, decoded); - if (totalBits + bits > 16 || !bits) { - TCacheEntry e = {static_cast<int>(DecodeCache.size()), 16 - totalBits}; - CacheEntries[i] = e; - break; - } - - for (TBuffer::TConstIterator it = decoded.Begin(); it != decoded.End(); ++it) { - DecodeCache.push_back(*it); - } - totalBits += bits; + buffer[1] = i >> 8; + buffer[0] = i; + NBitIO::TBitInput bin(buffer, buffer + sizeof(buffer)); + int totalBits = 0; + while (true) { + decoded.Resize(0); + int bits = codec.ReadNextChar(bin, decoded); + if (totalBits + bits > 16 || !bits) { + TCacheEntry e = {static_cast<int>(DecodeCache.size()), 16 - totalBits}; + CacheEntries[i] = e; + break; + } + + for (TBuffer::TConstIterator it = decoded.Begin(); it != decoded.End(); ++it) { + DecodeCache.push_back(*it); + } + totalBits += bits; } } - DecodeCache.push_back(0); - CacheEntries.shrink_to_fit(); - DecodeCache.shrink_to_fit(); + DecodeCache.push_back(0); + CacheEntries.shrink_to_fit(); + DecodeCache.shrink_to_fit(); } - void THuffmanCodec::TImpl::THuffmanCache::Decode(NBitIO::TBitInput& bin, TBuffer& out) const { - int bits = 0; - ui64 code = 0; - while (!bin.Eof()) { - ui64 f = 0; - const int toRead = 16 - bits; - if (toRead > 0 && bin.Read(f, toRead)) { - code = (code >> (16 - bits)) | (f << bits); - code &= 0xFFFF; - TCacheEntry entry = CacheEntries[code]; - int start = code > 0 ? CacheEntries[code - 1].EndOffset : 0; - out.Append((const char*)&DecodeCache[start], (const char*)&DecodeCache[entry.EndOffset]); - bits = entry.BitsLeft; - } else { // should never happen until there are exceptions or unaligned input - bin.Back(bits); - if (!Original.ReadNextChar(bin, out)) - break; - - code = 0; - bits = 0; - } + void THuffmanCodec::TImpl::THuffmanCache::Decode(NBitIO::TBitInput& bin, TBuffer& out) const { + int bits = 0; + ui64 code = 0; + while (!bin.Eof()) { + ui64 f = 0; + const int toRead = 16 - bits; + if (toRead > 0 && bin.Read(f, toRead)) { + code = (code >> (16 - bits)) | (f << bits); + code &= 0xFFFF; + TCacheEntry entry = CacheEntries[code]; + int start = code > 0 ? CacheEntries[code - 1].EndOffset : 0; + out.Append((const char*)&DecodeCache[start], (const char*)&DecodeCache[entry.EndOffset]); + bits = entry.BitsLeft; + } else { // should never happen until there are exceptions or unaligned input + bin.Back(bits); + if (!Original.ReadNextChar(bin, out)) + break; + + code = 0; + bits = 0; + } } } - THuffmanCodec::THuffmanCodec() - : Impl(new TImpl) - { - MyTraits.NeedsTraining = true; - MyTraits.PreservesPrefixGrouping = true; - MyTraits.PaddingBit = 1; - MyTraits.SizeOnEncodeMultiplier = 2; - MyTraits.SizeOnDecodeMultiplier = 8; - MyTraits.RecommendedSampleSize = 1 << 21; - } + THuffmanCodec::THuffmanCodec() + : Impl(new TImpl) + { + MyTraits.NeedsTraining = true; + MyTraits.PreservesPrefixGrouping = true; + MyTraits.PaddingBit = 1; + MyTraits.SizeOnEncodeMultiplier = 2; + MyTraits.SizeOnDecodeMultiplier = 8; + MyTraits.RecommendedSampleSize = 1 << 21; + } - THuffmanCodec::~THuffmanCodec() = default; + THuffmanCodec::~THuffmanCodec() = default; - ui8 THuffmanCodec::Encode(TStringBuf in, TBuffer& bbb) const { - if (Y_UNLIKELY(!Trained)) - ythrow TCodecException() << " not trained"; + ui8 THuffmanCodec::Encode(TStringBuf in, TBuffer& bbb) const { + if (Y_UNLIKELY(!Trained)) + ythrow TCodecException() << " not trained"; - return Impl->Encode(in, bbb); - } + return Impl->Encode(in, bbb); + } - void THuffmanCodec::Decode(TStringBuf in, TBuffer& bbb) const { - Impl->Decode(in, bbb); - } + void THuffmanCodec::Decode(TStringBuf in, TBuffer& bbb) const { + Impl->Decode(in, bbb); + } - void THuffmanCodec::Save(IOutputStream* out) const { - Impl->Save(out); - } + void THuffmanCodec::Save(IOutputStream* out) const { + Impl->Save(out); + } - void THuffmanCodec::Load(IInputStream* in) { - Impl->Load(in); - } + void THuffmanCodec::Load(IInputStream* in) { + Impl->Load(in); + } - void THuffmanCodec::DoLearn(ISequenceReader& in) { - Impl->Learn(&in); - } + void THuffmanCodec::DoLearn(ISequenceReader& in) { + Impl->Learn(&in); + } void THuffmanCodec::LearnByFreqs(const TArrayRef<std::pair<char, ui64>>& freqs) { - Impl->LearnByFreqs(freqs); - Trained = true; - } + Impl->LearnByFreqs(freqs); + Trained = true; + } } diff --git a/library/cpp/codecs/huffman_codec.h b/library/cpp/codecs/huffman_codec.h index 559545b90d..24f8397694 100644 --- a/library/cpp/codecs/huffman_codec.h +++ b/library/cpp/codecs/huffman_codec.h @@ -6,34 +6,34 @@ #include <util/string/cast.h> namespace NCodecs { - // for types greater than char, pipeline with TFreqCodec. + // for types greater than char, pipeline with TFreqCodec. - class THuffmanCodec: public ICodec { - class TImpl; - TIntrusivePtr<TImpl> Impl; + class THuffmanCodec: public ICodec { + class TImpl; + TIntrusivePtr<TImpl> Impl; - public: - THuffmanCodec(); - ~THuffmanCodec() override; + public: + THuffmanCodec(); + ~THuffmanCodec() override; - static TStringBuf MyName() { - return "huffman"; - } + static TStringBuf MyName() { + return "huffman"; + } - TString GetName() const override { + TString GetName() const override { return ToString(MyName()); - } + } - ui8 Encode(TStringBuf in, TBuffer& bbb) const override; + ui8 Encode(TStringBuf in, TBuffer& bbb) const override; - void Decode(TStringBuf in, TBuffer& bbb) const override; + void Decode(TStringBuf in, TBuffer& bbb) const override; void LearnByFreqs(const TArrayRef<std::pair<char, ui64>>& freqs); - protected: - void DoLearn(ISequenceReader& in) override; - void Save(IOutputStream* out) const override; - void Load(IInputStream* in) override; - }; + protected: + void DoLearn(ISequenceReader& in) override; + void Save(IOutputStream* out) const override; + void Load(IInputStream* in) override; + }; } diff --git a/library/cpp/codecs/pfor_codec.cpp b/library/cpp/codecs/pfor_codec.cpp index f6b3b0920b..d5dbc5a7fa 100644 --- a/library/cpp/codecs/pfor_codec.cpp +++ b/library/cpp/codecs/pfor_codec.cpp @@ -1,22 +1,22 @@ #include "pfor_codec.h" namespace NCodecs { - template <> - TStringBuf TPForCodec<ui64, true>::MyName() { - return "pfor-delta64-sorted"; - } - template <> - TStringBuf TPForCodec<ui32, true>::MyName() { - return "pfor-delta32-sorted"; - } + template <> + TStringBuf TPForCodec<ui64, true>::MyName() { + return "pfor-delta64-sorted"; + } + template <> + TStringBuf TPForCodec<ui32, true>::MyName() { + return "pfor-delta32-sorted"; + } - template <> - TStringBuf TPForCodec<ui64, false>::MyName() { - return "pfor-ui64"; - } - template <> - TStringBuf TPForCodec<ui32, false>::MyName() { - return "pfor-ui32"; - } + template <> + TStringBuf TPForCodec<ui64, false>::MyName() { + return "pfor-ui64"; + } + template <> + TStringBuf TPForCodec<ui32, false>::MyName() { + return "pfor-ui32"; + } } diff --git a/library/cpp/codecs/pfor_codec.h b/library/cpp/codecs/pfor_codec.h index d7d4bb8bf4..a1f2bf9f9a 100644 --- a/library/cpp/codecs/pfor_codec.h +++ b/library/cpp/codecs/pfor_codec.h @@ -10,202 +10,202 @@ #include <util/string/cast.h> namespace NCodecs { - template <typename T, bool WithDelta = false> - class TPForCodec: public ICodec { - using TUnsigned = std::make_unsigned_t<T>; - typedef TDeltaCodec<TUnsigned> TDCodec; + template <typename T, bool WithDelta = false> + class TPForCodec: public ICodec { + using TUnsigned = std::make_unsigned_t<T>; + typedef TDeltaCodec<TUnsigned> TDCodec; - typedef std::conditional_t<WithDelta, typename TDCodec::TDelta, T> TValue; - static_assert(std::is_unsigned<TValue>::value, "expect std:is_unsigned<TValue>::value"); + typedef std::conditional_t<WithDelta, typename TDCodec::TDelta, T> TValue; + static_assert(std::is_unsigned<TValue>::value, "expect std:is_unsigned<TValue>::value"); - static const ui64 BitsInT = sizeof(TUnsigned) * 8; + static const ui64 BitsInT = sizeof(TUnsigned) * 8; - TDCodec DeltaCodec; + TDCodec DeltaCodec; - public: - static TStringBuf MyName(); + public: + static TStringBuf MyName(); - TPForCodec() { - MyTraits.AssumesStructuredInput = true; - MyTraits.SizeOfInputElement = sizeof(T); - MyTraits.SizeOnDecodeMultiplier = sizeof(T); - } + TPForCodec() { + MyTraits.AssumesStructuredInput = true; + MyTraits.SizeOfInputElement = sizeof(T); + MyTraits.SizeOnDecodeMultiplier = sizeof(T); + } - TString GetName() const override { + TString GetName() const override { return ToString(MyName()); - } + } - ui8 Encode(TStringBuf s, TBuffer& b) const override { - b.Clear(); + ui8 Encode(TStringBuf s, TBuffer& b) const override { + b.Clear(); if (s.empty()) { - return 0; - } + return 0; + } b.Reserve(2 * s.size() + b.Size()); - if (WithDelta) { - auto buffer = TBufferTlsCache::TlsInstance().Item(); - TBuffer& db = buffer.Get(); - db.Clear(); + if (WithDelta) { + auto buffer = TBufferTlsCache::TlsInstance().Item(); + TBuffer& db = buffer.Get(); + db.Clear(); db.Reserve(2 * s.size()); - DeltaCodec.Encode(s, db); + DeltaCodec.Encode(s, db); s = TStringBuf{db.data(), db.size()}; - } + } TArrayRef<const TValue> tin{(const TValue*)s.data(), s.size() / sizeof(TValue)}; const ui64 sz = tin.size(); - ui64 bitcounts[BitsInT + 1]; - Zero(bitcounts); + ui64 bitcounts[BitsInT + 1]; + Zero(bitcounts); - ui32 zeros = 0; + ui32 zeros = 0; for (const TValue* it = tin.begin(); it != tin.end(); ++it) { - TUnsigned v = 1 + (TUnsigned)*it; - ui64 l = MostSignificantBit(v) + 1; - ++bitcounts[l]; - - if (!v) { - ++zeros; - } - } - - // cumulative bit counts - for (ui64 i = 0; i < BitsInT; ++i) { - bitcounts[i + 1] += bitcounts[i]; + TUnsigned v = 1 + (TUnsigned)*it; + ui64 l = MostSignificantBit(v) + 1; + ++bitcounts[l]; + + if (!v) { + ++zeros; + } + } + + // cumulative bit counts + for (ui64 i = 0; i < BitsInT; ++i) { + bitcounts[i + 1] += bitcounts[i]; } - bool hasexceptions = zeros; - ui64 optimalbits = BitsInT; + bool hasexceptions = zeros; + ui64 optimalbits = BitsInT; - { - ui64 excsize = 0; - ui64 minsize = sz * BitsInT; + { + ui64 excsize = 0; + ui64 minsize = sz * BitsInT; - for (ui64 current = BitsInT; current; --current) { - ui64 size = bitcounts[current] * current + (sz - bitcounts[current]) * (current + 6 + excsize) + zeros * (current + 6); + for (ui64 current = BitsInT; current; --current) { + ui64 size = bitcounts[current] * current + (sz - bitcounts[current]) * (current + 6 + excsize) + zeros * (current + 6); - excsize += current * bitcounts[current]; + excsize += current * bitcounts[current]; - if (size < minsize) { - minsize = size; - optimalbits = current; - hasexceptions = zeros || sz - bitcounts[current]; - } + if (size < minsize) { + minsize = size; + optimalbits = current; + hasexceptions = zeros || sz - bitcounts[current]; + } } } - if (!optimalbits || BitsInT == optimalbits) { - b.Append((ui8)-1); + if (!optimalbits || BitsInT == optimalbits) { + b.Append((ui8)-1); b.Append(s.data(), s.size()); - return 0; - } else { - NBitIO::TBitOutputVector<TBuffer> bout(&b); - bout.Write(0, 1); - bout.Write(hasexceptions, 1); - bout.Write(optimalbits, 6); + return 0; + } else { + NBitIO::TBitOutputVector<TBuffer> bout(&b); + bout.Write(0, 1); + bout.Write(hasexceptions, 1); + bout.Write(optimalbits, 6); for (const TValue* it = tin.begin(); it != tin.end(); ++it) { - TUnsigned word = 1 + (TUnsigned)*it; - ui64 len = MostSignificantBit(word) + 1; - if (len > optimalbits || !word) { - Y_ENSURE(hasexceptions, " "); - bout.Write(0, optimalbits); - bout.Write(len, 6); - bout.Write(word, len); - } else { - bout.Write(word, optimalbits); - } + TUnsigned word = 1 + (TUnsigned)*it; + ui64 len = MostSignificantBit(word) + 1; + if (len > optimalbits || !word) { + Y_ENSURE(hasexceptions, " "); + bout.Write(0, optimalbits); + bout.Write(len, 6); + bout.Write(word, len); + } else { + bout.Write(word, optimalbits); + } } - return bout.GetByteReminder(); - } // the rest of the last byte is zero padded. BitsInT is always > 7. + return bout.GetByteReminder(); + } // the rest of the last byte is zero padded. BitsInT is always > 7. } - void Decode(TStringBuf s, TBuffer& b) const override { - b.Clear(); + void Decode(TStringBuf s, TBuffer& b) const override { + b.Clear(); if (s.empty()) { - return; - } + return; + } b.Reserve(s.size() * sizeof(T) + b.Size()); - ui64 isplain = 0; - ui64 hasexceptions = 0; - ui64 bits = 0; - - NBitIO::TBitInput bin(s); - bin.ReadK<1>(isplain); - bin.ReadK<1>(hasexceptions); - bin.ReadK<6>(bits); - - if (Y_UNLIKELY(isplain)) { - s.Skip(1); - - if (WithDelta) { - DeltaCodec.Decode(s, b); - } else { + ui64 isplain = 0; + ui64 hasexceptions = 0; + ui64 bits = 0; + + NBitIO::TBitInput bin(s); + bin.ReadK<1>(isplain); + bin.ReadK<1>(hasexceptions); + bin.ReadK<6>(bits); + + if (Y_UNLIKELY(isplain)) { + s.Skip(1); + + if (WithDelta) { + DeltaCodec.Decode(s, b); + } else { b.Append(s.data(), s.size()); - } + } } else { - typename TDCodec::TDecoder decoder; + typename TDCodec::TDecoder decoder; - if (hasexceptions) { - ui64 word = 0; - while (bin.Read(word, bits)) { - if (word || (bin.ReadK<6>(word) && bin.Read(word, word))) { - --word; + if (hasexceptions) { + ui64 word = 0; + while (bin.Read(word, bits)) { + if (word || (bin.ReadK<6>(word) && bin.Read(word, word))) { + --word; - TValue t = word; + TValue t = word; - if (WithDelta) { - if (decoder.Decode(t)) { - TStringBuf r{(char*)&decoder.Result, sizeof(decoder.Result)}; + if (WithDelta) { + if (decoder.Decode(t)) { + TStringBuf r{(char*)&decoder.Result, sizeof(decoder.Result)}; b.Append(r.data(), r.size()); - } - } else { - TStringBuf r{(char*)&t, sizeof(t)}; + } + } else { + TStringBuf r{(char*)&t, sizeof(t)}; b.Append(r.data(), r.size()); } } } - } else { - ui64 word = 0; - T outarr[256 / sizeof(T)]; - ui32 cnt = 0; - while (true) { - ui64 v = bin.Read(word, bits); - - if ((!v) | (!word)) - break; - - --word; - TValue t = word; - - if (WithDelta) { - if (decoder.Decode(t)) { - outarr[cnt++] = decoder.Result; - } - } else { - outarr[cnt++] = t; - } - - if (cnt == Y_ARRAY_SIZE(outarr)) { - b.Append((const char*)outarr, sizeof(outarr)); - cnt = 0; + } else { + ui64 word = 0; + T outarr[256 / sizeof(T)]; + ui32 cnt = 0; + while (true) { + ui64 v = bin.Read(word, bits); + + if ((!v) | (!word)) + break; + + --word; + TValue t = word; + + if (WithDelta) { + if (decoder.Decode(t)) { + outarr[cnt++] = decoder.Result; + } + } else { + outarr[cnt++] = t; } + + if (cnt == Y_ARRAY_SIZE(outarr)) { + b.Append((const char*)outarr, sizeof(outarr)); + cnt = 0; + } } - if (cnt) { - b.Append((const char*)outarr, cnt * sizeof(T)); + if (cnt) { + b.Append((const char*)outarr, cnt * sizeof(T)); } } } } - protected: - void DoLearn(ISequenceReader&) override { - } - }; + protected: + void DoLearn(ISequenceReader&) override { + } + }; } diff --git a/library/cpp/codecs/sample.h b/library/cpp/codecs/sample.h index 15f03afcc5..5d3ab57f78 100644 --- a/library/cpp/codecs/sample.h +++ b/library/cpp/codecs/sample.h @@ -24,20 +24,20 @@ namespace NCodecs { } template <class TIter> - TStringBuf IterToStringBuf(TIter iter) { + TStringBuf IterToStringBuf(TIter iter) { return ValueToStringBuf(*iter); } template <class TItem> - class TSimpleSequenceReader: public ISequenceReader { + class TSimpleSequenceReader: public ISequenceReader { const TVector<TItem>& Items; size_t Idx = 0; public: TSimpleSequenceReader(const TVector<TItem>& items) : Items(items) - { - } + { + } bool NextRegion(TStringBuf& s) override { if (Idx >= Items.size()) { diff --git a/library/cpp/codecs/solar_codec.cpp b/library/cpp/codecs/solar_codec.cpp index d0692fe2a4..6c08b9e7bd 100644 --- a/library/cpp/codecs/solar_codec.cpp +++ b/library/cpp/codecs/solar_codec.cpp @@ -9,125 +9,125 @@ #include <util/ysaveload.h> namespace NCodecs { - static inline ui32 Append(TBuffer& pool, TStringBuf data) { + static inline ui32 Append(TBuffer& pool, TStringBuf data) { pool.Append(data.data(), data.size()); - return pool.Size(); - } - - void TSolarCodec::DoLearn(ISequenceReader& r) { - using namespace NGreedyDict; + return pool.Size(); + } - Decoder.clear(); - Pool.Clear(); + void TSolarCodec::DoLearn(ISequenceReader& r) { + using namespace NGreedyDict; - THolder<TEntrySet> set; + Decoder.clear(); + Pool.Clear(); - { - TMemoryPool pool(8112, TMemoryPool::TLinearGrow::Instance()); - TStringBufs bufs; + THolder<TEntrySet> set; - TStringBuf m; - while (r.NextRegion(m)) { - bufs.push_back(pool.AppendString(m)); - } + { + TMemoryPool pool(8112, TMemoryPool::TLinearGrow::Instance()); + TStringBufs bufs; - { - TDictBuilder b(Settings); - b.SetInput(bufs); - b.Build(MaxEntries, MaxIterations); + TStringBuf m; + while (r.NextRegion(m)) { + bufs.push_back(pool.AppendString(m)); + } - set = b.ReleaseEntrySet(); - } + { + TDictBuilder b(Settings); + b.SetInput(bufs); + b.Build(MaxEntries, MaxIterations); + + set = b.ReleaseEntrySet(); + } } - set->SetScores(ES_LEN_COUNT); - + set->SetScores(ES_LEN_COUNT); + { - TVector<std::pair<float, TStringBuf>> tmp; - tmp.reserve(set->size()); + TVector<std::pair<float, TStringBuf>> tmp; + tmp.reserve(set->size()); - for (const auto& it : *set) { - tmp.push_back(std::make_pair(-it.Score, TStringBuf(it.Str).Trunc(Max<ui32>() / Max<ui32>(MaxEntries, 1)))); - } + for (const auto& it : *set) { + tmp.push_back(std::make_pair(-it.Score, TStringBuf(it.Str).Trunc(Max<ui32>() / Max<ui32>(MaxEntries, 1)))); + } - Sort(tmp.begin(), tmp.end()); + Sort(tmp.begin(), tmp.end()); - Decoder.reserve(tmp.size() + 1); - Decoder.push_back(0); + Decoder.reserve(tmp.size() + 1); + Decoder.push_back(0); - for (const auto& it : tmp) { - Y_ENSURE(Decoder.back() == Pool.Size(), "learning invariant failed"); - ui32 endoff = Append(Pool, it.second); - Decoder.push_back(endoff); - } + for (const auto& it : tmp) { + Y_ENSURE(Decoder.back() == Pool.Size(), "learning invariant failed"); + ui32 endoff = Append(Pool, it.second); + Decoder.push_back(endoff); + } } - Pool.ShrinkToFit(); - Decoder.shrink_to_fit(); + Pool.ShrinkToFit(); + Decoder.shrink_to_fit(); - TBufferOutput bout; + TBufferOutput bout; - { - TVector<std::pair<TStringBuf, ui32>> tmp2; - tmp2.reserve(Decoder.size()); + { + TVector<std::pair<TStringBuf, ui32>> tmp2; + tmp2.reserve(Decoder.size()); - for (ui32 i = 1, sz = Decoder.size(); i < sz; ++i) { - TStringBuf s = DoDecode(i); - tmp2.push_back(std::make_pair(s, i - 1)); + for (ui32 i = 1, sz = Decoder.size(); i < sz; ++i) { + TStringBuf s = DoDecode(i); + tmp2.push_back(std::make_pair(s, i - 1)); Y_ENSURE(s.size() == (Decoder[i] - Decoder[i - 1]), "learning invariant failed"); - } + } - Sort(tmp2.begin(), tmp2.end()); + Sort(tmp2.begin(), tmp2.end()); - { - TEncoder::TBuilder builder(CTBF_PREFIX_GROUPED); - for (const auto& it : tmp2) { + { + TEncoder::TBuilder builder(CTBF_PREFIX_GROUPED); + for (const auto& it : tmp2) { builder.Add(it.first.data(), it.first.size(), it.second); - } + } - builder.Save(bout); + builder.Save(bout); } } - Encoder.Init(TBlob::FromBuffer(bout.Buffer())); - } - - void TSolarCodec::Save(IOutputStream* out) const { - TBlob b = Encoder.Data(); - ::Save(out, (ui32)b.Size()); - out->Write(b.Data(), b.Size()); + Encoder.Init(TBlob::FromBuffer(bout.Buffer())); } - void TSolarCodec::Load(IInputStream* in) { - ui32 sz; - ::Load(in, sz); - TLengthLimitedInput lin(in, sz); - Encoder.Init(TBlob::FromStream(lin)); - Pool.Clear(); - Decoder.clear(); - - TVector<std::pair<ui32, TString>> tmp; - - ui32 poolsz = 0; - for (TEncoder::TConstIterator it = Encoder.Begin(); it != Encoder.End(); ++it) { - const TString& s = it.GetKey(); - tmp.push_back(std::make_pair(it.GetValue(), !s ? TString("\0", 1) : s)); + void TSolarCodec::Save(IOutputStream* out) const { + TBlob b = Encoder.Data(); + ::Save(out, (ui32)b.Size()); + out->Write(b.Data(), b.Size()); + } + + void TSolarCodec::Load(IInputStream* in) { + ui32 sz; + ::Load(in, sz); + TLengthLimitedInput lin(in, sz); + Encoder.Init(TBlob::FromStream(lin)); + Pool.Clear(); + Decoder.clear(); + + TVector<std::pair<ui32, TString>> tmp; + + ui32 poolsz = 0; + for (TEncoder::TConstIterator it = Encoder.Begin(); it != Encoder.End(); ++it) { + const TString& s = it.GetKey(); + tmp.push_back(std::make_pair(it.GetValue(), !s ? TString("\0", 1) : s)); poolsz += Max<ui32>(s.size(), 1); - } + } - Sort(tmp.begin(), tmp.end()); + Sort(tmp.begin(), tmp.end()); - Pool.Reserve(poolsz); - Decoder.reserve(tmp.size() + 1); - Decoder.push_back(0); + Pool.Reserve(poolsz); + Decoder.reserve(tmp.size() + 1); + Decoder.push_back(0); - for (ui32 i = 0, sz2 = tmp.size(); i < sz2; ++i) { - Y_ENSURE(i == tmp[i].first, "oops! " << i << " " << tmp[i].first); - Decoder.push_back(Append(Pool, tmp[i].second)); - } + for (ui32 i = 0, sz2 = tmp.size(); i < sz2; ++i) { + Y_ENSURE(i == tmp[i].first, "oops! " << i << " " << tmp[i].first); + Decoder.push_back(Append(Pool, tmp[i].second)); + } - Pool.ShrinkToFit(); - Decoder.shrink_to_fit(); + Pool.ShrinkToFit(); + Decoder.shrink_to_fit(); } } diff --git a/library/cpp/codecs/solar_codec.h b/library/cpp/codecs/solar_codec.h index 7158ae7926..08fdf9d123 100644 --- a/library/cpp/codecs/solar_codec.h +++ b/library/cpp/codecs/solar_codec.h @@ -11,234 +11,234 @@ namespace NCodecs { // TODO: Попробовать добавлять в словарь вместе с намайненными словами также их суффиксы. // TODO: Возможно удастся, не слишком потеряв в сжатии, выиграть в робастности к небольшим изменениям в корпусе. - struct TVarIntTraits { - static const size_t MAX_VARINT32_BYTES = 5; - - static void Write(ui32 value, TBuffer& b) { - while (value > 0x7F) { - b.Append(static_cast<ui8>(value) | 0x80); - value >>= 7; - } - b.Append(static_cast<ui8>(value) & 0x7F); - } - - static void Read(TStringBuf& r, ui32& value) { - ui32 result = 0; - for (ui32 count = 0; count < MAX_VARINT32_BYTES; ++count) { - const ui32 b = static_cast<ui8>(r[0]); - r.Skip(1); - result |= static_cast<ui32>(b & 0x7F) << (7 * count); - if (!(b & 0x80)) { - value = result; - return; + struct TVarIntTraits { + static const size_t MAX_VARINT32_BYTES = 5; + + static void Write(ui32 value, TBuffer& b) { + while (value > 0x7F) { + b.Append(static_cast<ui8>(value) | 0x80); + value >>= 7; + } + b.Append(static_cast<ui8>(value) & 0x7F); + } + + static void Read(TStringBuf& r, ui32& value) { + ui32 result = 0; + for (ui32 count = 0; count < MAX_VARINT32_BYTES; ++count) { + const ui32 b = static_cast<ui8>(r[0]); + r.Skip(1); + result |= static_cast<ui32>(b & 0x7F) << (7 * count); + if (!(b & 0x80)) { + value = result; + return; } else if (Y_UNLIKELY(r.empty())) { - break; - } + break; + } } - Y_ENSURE_EX(false, TCodecException() << "Bad data"); + Y_ENSURE_EX(false, TCodecException() << "Bad data"); } - }; + }; - struct TShortIntTraits { - static const size_t SHORTINT_SIZE_LIMIT = 0x8000; + struct TShortIntTraits { + static const size_t SHORTINT_SIZE_LIMIT = 0x8000; - Y_FORCE_INLINE static void Write(ui32 value, TBuffer& b) { - Y_ENSURE_EX(value < SHORTINT_SIZE_LIMIT, TCodecException() << "Bad write method"); - if (value >= 0x80) { - b.Append(static_cast<ui8>(value >> 8) | 0x80); - } - b.Append(static_cast<ui8>(value)); + Y_FORCE_INLINE static void Write(ui32 value, TBuffer& b) { + Y_ENSURE_EX(value < SHORTINT_SIZE_LIMIT, TCodecException() << "Bad write method"); + if (value >= 0x80) { + b.Append(static_cast<ui8>(value >> 8) | 0x80); + } + b.Append(static_cast<ui8>(value)); } - Y_FORCE_INLINE static void Read(TStringBuf& r, ui32& value) { - ui32 result = static_cast<ui8>(r[0]); + Y_FORCE_INLINE static void Read(TStringBuf& r, ui32& value) { + ui32 result = static_cast<ui8>(r[0]); r.Skip(1); - if (result >= 0x80) { + if (result >= 0x80) { Y_ENSURE_EX(!r.empty(), TCodecException() << "Bad data"); - result = ((result << 8) & 0x7FFF) | static_cast<ui8>(r[0]); - r.Skip(1); - } - value = result; + result = ((result << 8) & 0x7FFF) | static_cast<ui8>(r[0]); + r.Skip(1); + } + value = result; } - }; + }; - class TSolarCodec: public ICodec { - public: - static TStringBuf MyName8k() { + class TSolarCodec: public ICodec { + public: + static TStringBuf MyName8k() { return TStringBuf("solar-8k"); - } - static TStringBuf MyName16k() { + } + static TStringBuf MyName16k() { return TStringBuf("solar-16k"); - } - static TStringBuf MyName32k() { + } + static TStringBuf MyName32k() { return TStringBuf("solar-32k"); - } - static TStringBuf MyName64k() { + } + static TStringBuf MyName64k() { return TStringBuf("solar-64k"); - } - static TStringBuf MyName256k() { + } + static TStringBuf MyName256k() { return TStringBuf("solar-256k"); - } - static TStringBuf MyName() { + } + static TStringBuf MyName() { return TStringBuf("solar"); - } - static TStringBuf MyName8kAdapt() { + } + static TStringBuf MyName8kAdapt() { return TStringBuf("solar-8k-a"); - } - static TStringBuf MyName16kAdapt() { + } + static TStringBuf MyName16kAdapt() { return TStringBuf("solar-16k-a"); - } - static TStringBuf MyName32kAdapt() { + } + static TStringBuf MyName32kAdapt() { return TStringBuf("solar-32k-a"); - } - static TStringBuf MyName64kAdapt() { + } + static TStringBuf MyName64kAdapt() { return TStringBuf("solar-64k-a"); - } - static TStringBuf MyName256kAdapt() { + } + static TStringBuf MyName256kAdapt() { return TStringBuf("solar-256k-a"); - } - static TStringBuf MyNameShortInt() { + } + static TStringBuf MyNameShortInt() { return TStringBuf("solar-si"); - } - - explicit TSolarCodec(ui32 maxentries = 1 << 14, ui32 maxiter = 16, const NGreedyDict::TBuildSettings& s = NGreedyDict::TBuildSettings()) - : Settings(s) - , MaxEntries(maxentries) - , MaxIterations(maxiter) - { - MyTraits.NeedsTraining = true; - MyTraits.SizeOnDecodeMultiplier = 2; - MyTraits.RecommendedSampleSize = maxentries * s.GrowLimit * maxiter * 8; - } - - ui8 /*free bits in last byte*/ Encode(TStringBuf r, TBuffer& b) const override { - EncodeImpl<TVarIntTraits>(r, b); - return 0; - } - - void Decode(TStringBuf r, TBuffer& b) const override { - DecodeImpl<TVarIntTraits>(r, b); - } - - TString GetName() const override { + } + + explicit TSolarCodec(ui32 maxentries = 1 << 14, ui32 maxiter = 16, const NGreedyDict::TBuildSettings& s = NGreedyDict::TBuildSettings()) + : Settings(s) + , MaxEntries(maxentries) + , MaxIterations(maxiter) + { + MyTraits.NeedsTraining = true; + MyTraits.SizeOnDecodeMultiplier = 2; + MyTraits.RecommendedSampleSize = maxentries * s.GrowLimit * maxiter * 8; + } + + ui8 /*free bits in last byte*/ Encode(TStringBuf r, TBuffer& b) const override { + EncodeImpl<TVarIntTraits>(r, b); + return 0; + } + + void Decode(TStringBuf r, TBuffer& b) const override { + DecodeImpl<TVarIntTraits>(r, b); + } + + TString GetName() const override { return ToString(MyName()); - } + } - protected: - void DoLearn(ISequenceReader&) override; - void Save(IOutputStream*) const override; - void Load(IInputStream*) override; + protected: + void DoLearn(ISequenceReader&) override; + void Save(IOutputStream*) const override; + void Load(IInputStream*) override; - Y_FORCE_INLINE TStringBuf SubStr(ui32 begoff, ui32 endoff) const { - return TStringBuf(Pool.Data() + begoff, endoff - begoff); - } + Y_FORCE_INLINE TStringBuf SubStr(ui32 begoff, ui32 endoff) const { + return TStringBuf(Pool.Data() + begoff, endoff - begoff); + } - Y_FORCE_INLINE TStringBuf DoDecode(ui32 num) const { - return SubStr(Decoder[num - 1], Decoder[num]); - } + Y_FORCE_INLINE TStringBuf DoDecode(ui32 num) const { + return SubStr(Decoder[num - 1], Decoder[num]); + } - template <class TTraits> - Y_FORCE_INLINE void EncodeImpl(TStringBuf r, TBuffer& b) const { - b.Clear(); + template <class TTraits> + Y_FORCE_INLINE void EncodeImpl(TStringBuf r, TBuffer& b) const { + b.Clear(); b.Reserve(r.size()); while (!r.empty()) { - size_t sz = 0; - ui32 val = (ui32)-1; - Encoder.FindLongestPrefix(r, &sz, &val); - TTraits::Write(val + 1, b); - r.Skip(Max<size_t>(sz, 1)); - } + size_t sz = 0; + ui32 val = (ui32)-1; + Encoder.FindLongestPrefix(r, &sz, &val); + TTraits::Write(val + 1, b); + r.Skip(Max<size_t>(sz, 1)); + } } - template <class TTraits> - Y_FORCE_INLINE void DecodeImpl(TStringBuf r, TBuffer& b) const { - b.Clear(); + template <class TTraits> + Y_FORCE_INLINE void DecodeImpl(TStringBuf r, TBuffer& b) const { + b.Clear(); b.Reserve(r.size()); - ui32 v = 0; + ui32 v = 0; while (!r.empty()) { - TTraits::Read(r, v); - TStringBuf s = DoDecode(v); + TTraits::Read(r, v); + TStringBuf s = DoDecode(v); b.Append(s.data(), s.size()); - } - } - - inline bool CanUseShortInt() const { - return Decoder.size() < TShortIntTraits::SHORTINT_SIZE_LIMIT; - } - - private: - typedef TCompactTrie<char, ui32> TEncoder; - typedef TVector<ui32> TDecoder; - - TBuffer Pool; - TEncoder Encoder; - TDecoder Decoder; - - NGreedyDict::TBuildSettings Settings; - ui32 MaxEntries; - ui32 MaxIterations; - }; - - // Uses varints or shortints depending on the decoder size - class TAdaptiveSolarCodec: public TSolarCodec { - public: - explicit TAdaptiveSolarCodec(ui32 maxentries = 1 << 14, ui32 maxiter = 16, const NGreedyDict::TBuildSettings& s = NGreedyDict::TBuildSettings()) - : TSolarCodec(maxentries, maxiter, s) - { - } - - ui8 /*free bits in last byte*/ Encode(TStringBuf r, TBuffer& b) const override { - if (CanUseShortInt()) { - EncodeImpl<TShortIntTraits>(r, b); - } else { - EncodeImpl<TVarIntTraits>(r, b); - } - - return 0; - } - - void Decode(TStringBuf r, TBuffer& b) const override { - if (CanUseShortInt()) { - DecodeImpl<TShortIntTraits>(r, b); - } else { - DecodeImpl<TVarIntTraits>(r, b); - } - } - - TString GetName() const override { - if (CanUseShortInt()) { + } + } + + inline bool CanUseShortInt() const { + return Decoder.size() < TShortIntTraits::SHORTINT_SIZE_LIMIT; + } + + private: + typedef TCompactTrie<char, ui32> TEncoder; + typedef TVector<ui32> TDecoder; + + TBuffer Pool; + TEncoder Encoder; + TDecoder Decoder; + + NGreedyDict::TBuildSettings Settings; + ui32 MaxEntries; + ui32 MaxIterations; + }; + + // Uses varints or shortints depending on the decoder size + class TAdaptiveSolarCodec: public TSolarCodec { + public: + explicit TAdaptiveSolarCodec(ui32 maxentries = 1 << 14, ui32 maxiter = 16, const NGreedyDict::TBuildSettings& s = NGreedyDict::TBuildSettings()) + : TSolarCodec(maxentries, maxiter, s) + { + } + + ui8 /*free bits in last byte*/ Encode(TStringBuf r, TBuffer& b) const override { + if (CanUseShortInt()) { + EncodeImpl<TShortIntTraits>(r, b); + } else { + EncodeImpl<TVarIntTraits>(r, b); + } + + return 0; + } + + void Decode(TStringBuf r, TBuffer& b) const override { + if (CanUseShortInt()) { + DecodeImpl<TShortIntTraits>(r, b); + } else { + DecodeImpl<TVarIntTraits>(r, b); + } + } + + TString GetName() const override { + if (CanUseShortInt()) { return ToString(MyNameShortInt()); - } else { + } else { return ToString(MyName()); - } + } } - }; + }; - class TSolarCodecShortInt: public TSolarCodec { - public: - explicit TSolarCodecShortInt(ui32 maxentries = 1 << 14, ui32 maxiter = 16, const NGreedyDict::TBuildSettings& s = NGreedyDict::TBuildSettings()) - : TSolarCodec(maxentries, maxiter, s) - { + class TSolarCodecShortInt: public TSolarCodec { + public: + explicit TSolarCodecShortInt(ui32 maxentries = 1 << 14, ui32 maxiter = 16, const NGreedyDict::TBuildSettings& s = NGreedyDict::TBuildSettings()) + : TSolarCodec(maxentries, maxiter, s) + { } - ui8 /*free bits in last byte*/ Encode(TStringBuf r, TBuffer& b) const override { - EncodeImpl<TShortIntTraits>(r, b); - return 0; - } + ui8 /*free bits in last byte*/ Encode(TStringBuf r, TBuffer& b) const override { + EncodeImpl<TShortIntTraits>(r, b); + return 0; + } - void Decode(TStringBuf r, TBuffer& b) const override { - DecodeImpl<TShortIntTraits>(r, b); - } + void Decode(TStringBuf r, TBuffer& b) const override { + DecodeImpl<TShortIntTraits>(r, b); + } - TString GetName() const override { + TString GetName() const override { return ToString(MyNameShortInt()); - } - - protected: - void Load(IInputStream* in) override { - TSolarCodec::Load(in); - Y_ENSURE_EX(CanUseShortInt(), TCodecException() << "Bad data"); - } - }; + } + + protected: + void Load(IInputStream* in) override { + TSolarCodec::Load(in); + Y_ENSURE_EX(CanUseShortInt(), TCodecException() << "Bad data"); + } + }; } diff --git a/library/cpp/codecs/static/builder.h b/library/cpp/codecs/static/builder.h index d7533be4d5..ece4dfa529 100644 --- a/library/cpp/codecs/static/builder.h +++ b/library/cpp/codecs/static/builder.h @@ -19,7 +19,7 @@ namespace NCodecs { time_t Timestamp = TInstant::Now().TimeT(); TString RevisionInfo = (TStringBuilder() << "r" << ToString(GetProgramSvnRevision())); TString TrainingSetComment; // a human comment on the training data - TString TrainingSetResId; // sandbox resid of the training set + TString TrainingSetResId; // sandbox resid of the training set }; TStaticCodecInfo BuildStaticCodec(const TVector<TString>& trainingData, const TCodecBuildInfo&); diff --git a/library/cpp/codecs/static/example/example.cpp b/library/cpp/codecs/static/example/example.cpp index 5b750b717e..b0566a8c2e 100644 --- a/library/cpp/codecs/static/example/example.cpp +++ b/library/cpp/codecs/static/example/example.cpp @@ -5,10 +5,10 @@ #include <util/generic/yexception.h> extern "C" { -extern const ui8 codec_info_huff_20160707[]; -extern const ui32 codec_info_huff_20160707Size; -extern const ui8 codec_info_sa_huff_20160707[]; -extern const ui32 codec_info_sa_huff_20160707Size; +extern const ui8 codec_info_huff_20160707[]; +extern const ui32 codec_info_huff_20160707Size; +extern const ui8 codec_info_sa_huff_20160707[]; +extern const ui32 codec_info_sa_huff_20160707Size; }; namespace NStaticCodecExample { diff --git a/library/cpp/codecs/static/example/example.h b/library/cpp/codecs/static/example/example.h index f9b3a7324b..41003fb187 100644 --- a/library/cpp/codecs/static/example/example.h +++ b/library/cpp/codecs/static/example/example.h @@ -4,11 +4,11 @@ #include <util/generic/buffer.h> namespace NStaticCodecExample { - enum EDictVersion : ui8 { - DV_NULL = 0, - DV_HUFF_20160707, - DV_SA_HUFF_20160707, - DV_COUNT + enum EDictVersion : ui8 { + DV_NULL = 0, + DV_HUFF_20160707, + DV_SA_HUFF_20160707, + DV_COUNT }; void Encode(TBuffer&, TStringBuf, EDictVersion dv = DV_SA_HUFF_20160707); diff --git a/library/cpp/codecs/static/static.cpp b/library/cpp/codecs/static/static.cpp index 44a07dd73a..97ddbd8364 100644 --- a/library/cpp/codecs/static/static.cpp +++ b/library/cpp/codecs/static/static.cpp @@ -69,8 +69,8 @@ namespace NCodecs { s << "sample mult: " << ci.GetDebugInfo().GetSampleSizeMultiplier() << Endl; s << "orig.compress: " << ci.GetDebugInfo().GetCompression() * 100 << " %" << Endl; s << "timestamp: " << ci.GetDebugInfo().GetTimestamp() << " (" - << NDatetime::TSimpleTM::NewLocal(ci.GetDebugInfo().GetTimestamp()).ToString() - << ")" << Endl; + << NDatetime::TSimpleTM::NewLocal(ci.GetDebugInfo().GetTimestamp()).ToString() + << ")" << Endl; s << "revision: " << ci.GetDebugInfo().GetRevisionInfo() << Endl; s << "training set comment: " << ci.GetDebugInfo().GetTrainingSetComment() << Endl; s << "training set resId: " << ci.GetDebugInfo().GetTrainingSetResId() << Endl; diff --git a/library/cpp/codecs/static/tools/static_codec_checker/static_codec_checker.cpp b/library/cpp/codecs/static/tools/static_codec_checker/static_codec_checker.cpp index 9c8d568d82..3668a7583a 100644 --- a/library/cpp/codecs/static/tools/static_codec_checker/static_codec_checker.cpp +++ b/library/cpp/codecs/static/tools/static_codec_checker/static_codec_checker.cpp @@ -22,17 +22,17 @@ int main(int argc, char** argv) { opts.SetCmdLineDescr("-c 9089f3e9b7a0f0d4.codec_info -t -f base64 qtrees.sample.txt"); NCodecs::TStaticCodecInfo codec; - opts.AddLongOption('c', "codec-info").RequiredArgument("codec_info").Handler1T<TString>([&codecFile, &codec, &codecPtr](TString name) { - codecFile = name; - codec.CopyFrom(NCodecs::LoadCodecInfoFromString(TUnbufferedFileInput(name).ReadAll())); - codecPtr = NCodecs::ICodec::RestoreFromString(codec.GetStoredCodec()); - }) - .Required() - .Help(".codec_info file with serialized static data for codec"); + opts.AddLongOption('c', "codec-info").RequiredArgument("codec_info").Handler1T<TString>([&codecFile, &codec, &codecPtr](TString name) { + codecFile = name; + codec.CopyFrom(NCodecs::LoadCodecInfoFromString(TUnbufferedFileInput(name).ReadAll())); + codecPtr = NCodecs::ICodec::RestoreFromString(codec.GetStoredCodec()); + }) + .Required() + .Help(".codec_info file with serialized static data for codec"); - opts.AddLongOption('t', "test").NoArgument().StoreValue(&testCompression, true).Optional().Help("test current performance"); + opts.AddLongOption('t', "test").NoArgument().StoreValue(&testCompression, true).Optional().Help("test current performance"); - opts.AddLongOption('f', "format").RequiredArgument(TStringBuilder() << "(" << NCodecs::DSF_PLAIN_LF << "|" << NCodecs::DSF_BASE64_LF << ")").StoreResult(&fmt).Optional().Help("test set input file format"); + opts.AddLongOption('f', "format").RequiredArgument(TStringBuilder() << "(" << NCodecs::DSF_PLAIN_LF << "|" << NCodecs::DSF_BASE64_LF << ")").StoreResult(&fmt).Optional().Help("test set input file format"); opts.SetFreeArgsMin(0); opts.SetFreeArgTitle(0, "testing_set_input_file", "testing set input files"); diff --git a/library/cpp/codecs/static/tools/static_codec_generator/static_codec_generator.cpp b/library/cpp/codecs/static/tools/static_codec_generator/static_codec_generator.cpp index 45fdb5c5fe..073689737d 100644 --- a/library/cpp/codecs/static/tools/static_codec_generator/static_codec_generator.cpp +++ b/library/cpp/codecs/static/tools/static_codec_generator/static_codec_generator.cpp @@ -17,26 +17,26 @@ int main(int argc, char** argv) { opts.SetCmdLineDescr("-m 'Training set: 100000 qtrees taken from web mmeta logs' -f base64 qtrees.sample.txt"); opts.SetTitle("Teaches the codec and serializes it as a file named CODECNAME.hash(CODECDATA).bin"); - opts.AddLongOption('m', "message").RequiredArgument("training_set_comment").StoreResult(&info.TrainingSetComment).Required().Help("a human description for the training set"); + opts.AddLongOption('m', "message").RequiredArgument("training_set_comment").StoreResult(&info.TrainingSetComment).Required().Help("a human description for the training set"); - opts.AddLongOption('r', "resource").RequiredArgument("training_set_res_id").StoreResult(&info.TrainingSetResId).Optional().Help("sandbox resource id for the training set"); + opts.AddLongOption('r', "resource").RequiredArgument("training_set_res_id").StoreResult(&info.TrainingSetResId).Optional().Help("sandbox resource id for the training set"); - opts.AddLongOption('c', "codec").RequiredArgument("codec_name").StoreResult(&info.CodecName).Optional().DefaultValue(info.CodecName); + opts.AddLongOption('c', "codec").RequiredArgument("codec_name").StoreResult(&info.CodecName).Optional().DefaultValue(info.CodecName); - opts.AddLongOption('s', "sample-multiplier").RequiredArgument("multiplier").StoreResult(&info.SampleSizeMultiplier).Optional().DefaultValue(ToString(info.SampleSizeMultiplier)).Help("multiplier for default sample size"); + opts.AddLongOption('s', "sample-multiplier").RequiredArgument("multiplier").StoreResult(&info.SampleSizeMultiplier).Optional().DefaultValue(ToString(info.SampleSizeMultiplier)).Help("multiplier for default sample size"); - opts.AddLongOption('f', "format").RequiredArgument(TStringBuilder() << "(" << NCodecs::DSF_PLAIN_LF << "|" << NCodecs::DSF_BASE64_LF << ")").StoreResult(&fmt).Required().Help("training set input file format"); + opts.AddLongOption('f', "format").RequiredArgument(TStringBuilder() << "(" << NCodecs::DSF_PLAIN_LF << "|" << NCodecs::DSF_BASE64_LF << ")").StoreResult(&fmt).Required().Help("training set input file format"); - opts.AddLongOption("list-codecs").NoArgument().Handler0([]() { - Cout << JoinStrings(NCodecs::ICodec::GetCodecsList(), "\n") << Endl; - exit(0); - }) - .Optional() - .Help("list available codecs"); + opts.AddLongOption("list-codecs").NoArgument().Handler0([]() { + Cout << JoinStrings(NCodecs::ICodec::GetCodecsList(), "\n") << Endl; + exit(0); + }) + .Optional() + .Help("list available codecs"); - opts.AddLongOption("fake-revision").RequiredArgument("revision").StoreResult(&info.RevisionInfo).Optional().Hidden(); // replace static_codec_generator revision in debug info + opts.AddLongOption("fake-revision").RequiredArgument("revision").StoreResult(&info.RevisionInfo).Optional().Hidden(); // replace static_codec_generator revision in debug info - opts.AddLongOption("fake-timestamp").RequiredArgument("timestamp").StoreResult(&info.Timestamp).Optional().Hidden(); // replace generating timestamp in debug info + opts.AddLongOption("fake-timestamp").RequiredArgument("timestamp").StoreResult(&info.Timestamp).Optional().Hidden(); // replace generating timestamp in debug info opts.SetFreeArgsMin(0); opts.SetFreeArgTitle(0, "training_set_input_file", "training set input files"); diff --git a/library/cpp/codecs/static/ut/builder_ut.cpp b/library/cpp/codecs/static/ut/builder_ut.cpp index b47c279ed1..d9c4eb9e47 100644 --- a/library/cpp/codecs/static/ut/builder_ut.cpp +++ b/library/cpp/codecs/static/ut/builder_ut.cpp @@ -3,11 +3,11 @@ #include <library/cpp/codecs/static/static_codec_info.pb.h> #include <util/string/vector.h> -class TStaticCodecInfoBuilderTest: public NUnitTest::TTestBase { +class TStaticCodecInfoBuilderTest: public NUnitTest::TTestBase { UNIT_TEST_SUITE(TStaticCodecInfoBuilderTest) - UNIT_TEST(TestBuild) + UNIT_TEST(TestBuild) UNIT_TEST_SUITE_END(); - + private: TVector<TString> PrepareData() { TVector<TString> data; diff --git a/library/cpp/codecs/static/ut/static_ut.cpp b/library/cpp/codecs/static/ut/static_ut.cpp index 57e1e62887..315e1bf0b0 100644 --- a/library/cpp/codecs/static/ut/static_ut.cpp +++ b/library/cpp/codecs/static/ut/static_ut.cpp @@ -1,11 +1,11 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/codecs/static/example/example.h> -class TStaticCodecUsageTest: public NUnitTest::TTestBase { +class TStaticCodecUsageTest: public NUnitTest::TTestBase { UNIT_TEST_SUITE(TStaticCodecUsageTest) - UNIT_TEST(TestUsage) + UNIT_TEST(TestUsage) UNIT_TEST_SUITE_END(); - + private: void DoTestUsage(NStaticCodecExample::EDictVersion dv, size_t expectedSize) { const TStringBuf letov = "Всё идёт по плану"; diff --git a/library/cpp/codecs/tls_cache.h b/library/cpp/codecs/tls_cache.h index 0184e4bb6c..7068ea333f 100644 --- a/library/cpp/codecs/tls_cache.h +++ b/library/cpp/codecs/tls_cache.h @@ -15,15 +15,15 @@ namespace NCodecs { } }; - template <class TItem, class TCleaner = TClear<TItem>> + template <class TItem, class TCleaner = TClear<TItem>> class TTlsCache { using TSelf = TTlsCache<TItem, TCleaner>; - struct TItemHolder: public TIntrusiveListItem<TItemHolder> { + struct TItemHolder: public TIntrusiveListItem<TItemHolder> { TItemHolder(TSelf& factory) : Factory(factory) - { - } + { + } void Release() { Factory.Release(*this); @@ -37,14 +37,14 @@ namespace NCodecs { public: explicit TItemGuard(TSelf& fact) : Holder(fact.Acquire()) - { - } + { + } - TItemGuard(TItemGuard&& other) noexcept { + TItemGuard(TItemGuard&& other) noexcept { *this = std::move(other); } - TItemGuard& operator=(TItemGuard&& other) noexcept { + TItemGuard& operator=(TItemGuard&& other) noexcept { if (&other != this) { std::swap(Holder, other.Holder); } diff --git a/library/cpp/codecs/ut/codecs_ut.cpp b/library/cpp/codecs/ut/codecs_ut.cpp index caf6089aef..36675f6b63 100644 --- a/library/cpp/codecs/ut/codecs_ut.cpp +++ b/library/cpp/codecs/ut/codecs_ut.cpp @@ -13,107 +13,107 @@ #include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> namespace { - const char* TextValues[] = { - "! сентября газета", - "!(возмездие это)!", - "!(материнский капитал)", - "!(пермь березники)", - "!биография | !жизнь / + розинг | зворыгин & изобретение | телевидение | электронно лучевая трубка", - "!овсиенко николай павлович", - "!путин", - "\"i'm on you\" p. diddy тимати клип", - "\"билайн\" представит собственный планшет", - "\"в особо крупном размере\"", - "\"викиликс\" джулиан ассанж", - "\"вимм билль данн", - "\"газэнергосеть астрахань", - "\"газэнергосеть астрахань\"", - "\"домодедово\" ту-154", - "\"жилина\" \"спартак\" видео", - "\"зелёнsq шершнm\"", - "\"зелёного шершня\"", - "\"золотой граммофон\" марины яблоковой", - "\"золотой граммофон-2010\"", - "\"калинниковы\"", - "\"манчестер юнайтед\" (англия) \"валенсия\" (испания) 1:1 (0:1)", - "\"маркер\"", - "\"моника\" засыпает москву снегом", - "\"моника\" снегопад", - "\"о безопасности\",", - "\"памятку\" для пассажиров воздушных международных рейсов", - "\"петровский парк\" и \"ходынское поле\"", - "\"путинская\" трава", - "\"пятерочка\"купила \"копейку\"", - "\"пятёрочка\" и \"копейка\" объединились", - "\"реал\" \"осер\" 4:0", - "\"речь мутко\"", - "\"российский лес 2010\"", - "\"ростехинвентаризация федеральное бти\" рубцов", - "\"саня останется с нами\",", - "\"следопыт\" реалити шоу", - "\"слышишь\" молодые авторы", - "\"стадион\"", - "\"ходынское поле\" метро", - "\"хроники нарнии\"", - "\"чистая вода\"", - "\"школа деда мороза\"", - "# asus -1394", - "# сторонники wikileaks", - "#106#", - "#11", - "#8 какой цвет", - "#если клиент", - "$ 13,79", - "$ xnj ,s dct ,skb ljdjkmys !!!", - "$ в день", - "$ диск компьютера", - "$.ajax", - "$125 000", - "$курс", - "% в си", - "% влады", - "% годовых", - "% женщин и % мужчин в россии", - "% занятости персонала", - "% инфляции 2010", - "% инфляции в 2010 г.", - "% налога", - "% налогов в 2010г.", - "% общего количества", - "% от числа", - "% по налогу на прибыль организации", - "%24", - "%академия%", - "%комарова%татьяна", - "& в 1с", - "&& (+не существует | !такой проблемы)", - ">>>скачать | download c cs strikez.clan.su<<<", - ">hbq nbityrjd", - "< какой знак", - "< лицей | < техническая школа# < история#< лицей сегодня#< перечень профессий#< руководство лицея#< прием учащихся#< контакты#< схема проезда#< фотогалереяистория создания лицея и основные этапы путиулица купчинская дом 28", - "<<link>>", - "</storage>", - "<bfnkjy", - "<bktntd", - "<cr", - "<ddr3>", - "<e[ufknthcrbq abyfycjdsq", - "<fcctqys", - "<fhcf", - "<fhctkjyf he,by", - "<firbhbz", - "<fyr djphj;ltybt", - "<fyr vjcrds", - "<fyr резерв", - "<fyufkjh", - "<index>", - "<jkmifz jrhe;yfz rbtd", - "<kbpytws", - "<megafon> интернет", - "<thtpybrb gthvcrbq rhfq", - "<tkjxrf", - "<беларусь это мы", - "<бокс, версия ibf", + const char* TextValues[] = { + "! сентября газета", + "!(возмездие это)!", + "!(материнский капитал)", + "!(пермь березники)", + "!биография | !жизнь / + розинг | зворыгин & изобретение | телевидение | электронно лучевая трубка", + "!овсиенко николай павлович", + "!путин", + "\"i'm on you\" p. diddy тимати клип", + "\"билайн\" представит собственный планшет", + "\"в особо крупном размере\"", + "\"викиликс\" джулиан ассанж", + "\"вимм билль данн", + "\"газэнергосеть астрахань", + "\"газэнергосеть астрахань\"", + "\"домодедово\" ту-154", + "\"жилина\" \"спартак\" видео", + "\"зелёнsq шершнm\"", + "\"зелёного шершня\"", + "\"золотой граммофон\" марины яблоковой", + "\"золотой граммофон-2010\"", + "\"калинниковы\"", + "\"манчестер юнайтед\" (англия) \"валенсия\" (испания) 1:1 (0:1)", + "\"маркер\"", + "\"моника\" засыпает москву снегом", + "\"моника\" снегопад", + "\"о безопасности\",", + "\"памятку\" для пассажиров воздушных международных рейсов", + "\"петровский парк\" и \"ходынское поле\"", + "\"путинская\" трава", + "\"пятерочка\"купила \"копейку\"", + "\"пятёрочка\" и \"копейка\" объединились", + "\"реал\" \"осер\" 4:0", + "\"речь мутко\"", + "\"российский лес 2010\"", + "\"ростехинвентаризация федеральное бти\" рубцов", + "\"саня останется с нами\",", + "\"следопыт\" реалити шоу", + "\"слышишь\" молодые авторы", + "\"стадион\"", + "\"ходынское поле\" метро", + "\"хроники нарнии\"", + "\"чистая вода\"", + "\"школа деда мороза\"", + "# asus -1394", + "# сторонники wikileaks", + "#106#", + "#11", + "#8 какой цвет", + "#если клиент", + "$ 13,79", + "$ xnj ,s dct ,skb ljdjkmys !!!", + "$ в день", + "$ диск компьютера", + "$.ajax", + "$125 000", + "$курс", + "% в си", + "% влады", + "% годовых", + "% женщин и % мужчин в россии", + "% занятости персонала", + "% инфляции 2010", + "% инфляции в 2010 г.", + "% налога", + "% налогов в 2010г.", + "% общего количества", + "% от числа", + "% по налогу на прибыль организации", + "%24", + "%академия%", + "%комарова%татьяна", + "& в 1с", + "&& (+не существует | !такой проблемы)", + ">>>скачать | download c cs strikez.clan.su<<<", + ">hbq nbityrjd", + "< какой знак", + "< лицей | < техническая школа# < история#< лицей сегодня#< перечень профессий#< руководство лицея#< прием учащихся#< контакты#< схема проезда#< фотогалереяистория создания лицея и основные этапы путиулица купчинская дом 28", + "<<link>>", + "</storage>", + "<bfnkjy", + "<bktntd", + "<cr", + "<ddr3>", + "<e[ufknthcrbq abyfycjdsq", + "<fcctqys", + "<fhcf", + "<fhctkjyf he,by", + "<firbhbz", + "<fyr djphj;ltybt", + "<fyr vjcrds", + "<fyr резерв", + "<fyufkjh", + "<index>", + "<jkmifz jrhe;yfz rbtd", + "<kbpytws", + "<megafon> интернет", + "<thtpybrb gthvcrbq rhfq", + "<tkjxrf", + "<беларусь это мы", + "<бокс, версия ibf", "designer tree svc", "seriesg810", "doll makers", @@ -854,11 +854,11 @@ namespace { "resume maker", "lymphomatoid papulosis", "sez.com", - }; + }; } class TCodecsTest: public TTestBase { - UNIT_TEST_SUITE(TCodecsTest); + UNIT_TEST_SUITE(TCodecsTest); UNIT_TEST(TestPipeline) UNIT_TEST(TestDelta) UNIT_TEST(TestHuffman) @@ -869,14 +869,14 @@ class TCodecsTest: public TTestBase { UNIT_TEST(TestPFor) UNIT_TEST(TestRegistry) - UNIT_TEST_SUITE_END(); + UNIT_TEST_SUITE_END(); private: TString PrintError(TStringBuf learn, TStringBuf test, TStringBuf codec, ui32 i) { TString s; TStringOutput sout(s); - sout << codec << ": " << i << ", " - << "\n"; + sout << codec << ": " << i << ", " + << "\n"; sout << HexEncode(learn.data(), learn.size()); //NEscJ::EscapeJ<true>(learn, sout); sout << " != \n"; sout << HexEncode(test.data(), test.size()); //NEscJ::EscapeJ<true>(test, sout); @@ -1009,8 +1009,8 @@ private: AppendTo(d.back(), -1LL); AppendTo(d.back(), -1LL); - TestCodec<TDeltaCodec<ui64, true>, false>(d); - TestCodec<TDeltaCodec<ui64, false>, false>(d); + TestCodec<TDeltaCodec<ui64, true>, false>(d); + TestCodec<TDeltaCodec<ui64, false>, false>(d); } void TestPFor() { @@ -1050,7 +1050,7 @@ private: AppendTo(d.back(), -1LL); AppendTo(d.back(), -2LL); - TestCodec<TPForCodec<ui64>, false>(d); + TestCodec<TPForCodec<ui64>, false>(d); TestCodec<TPForCodec<ui64, true>, true>(d); } { @@ -1080,7 +1080,7 @@ private: AppendTo(d.back(), -1); AppendTo(d.back(), -2); - TestCodec<TPForCodec<ui32>, false>(d); + TestCodec<TPForCodec<ui32>, false>(d); TestCodec<TPForCodec<ui32, true>, false>(d); } { @@ -1326,7 +1326,7 @@ private: } TestCodec<TPipelineCodec, true>(learn, test, - new TPipelineCodec(new TSolarCodec(512, 8), new TSolarCodec(512, 8), new THuffmanCodec)); + new TPipelineCodec(new TSolarCodec(512, 8), new TSolarCodec(512, 8), new THuffmanCodec)); } { TVector<TBuffer> d; @@ -1338,7 +1338,7 @@ private: } TestCodec<TPipelineCodec, false>(d, TVector<TBuffer>(), - new TPipelineCodec(new TDeltaCodec<ui32, false>, new TPForCodec<ui32>)); + new TPipelineCodec(new TDeltaCodec<ui32, false>, new TPForCodec<ui32>)); } } diff --git a/library/cpp/codecs/ut/float_huffman_ut.cpp b/library/cpp/codecs/ut/float_huffman_ut.cpp index 3156fb1f46..dddff22173 100644 --- a/library/cpp/codecs/ut/float_huffman_ut.cpp +++ b/library/cpp/codecs/ut/float_huffman_ut.cpp @@ -60,7 +60,7 @@ Y_UNIT_TEST_SUITE(FloatHuffmanTest) { 0.517876, 0.145833, 0.372549, 0, 0.991667, 0.602125, 0.161979, 0, 0, 0, 0, 0.0255146, 0.947855, 0, 0, 0, 0, 0, 0, 0, 0, 0.847059, 0.679841, 0, 0.156863, 0, 0, 1, 0, 0, 0, 0, 0.969697, 0, 0, 0.564706, 0, 0, 0, 0, 0, 1, 0.0367282, 0.0395228, 0, 0, 0, - 0, 0, 0.0470588, 0.141176, 0.054902, 0, 0, 0, 0}; + 0, 0, 0.0470588, 0.141176, 0.054902, 0, 0, 0, 0}; static const size_t FactorCount = Y_ARRAY_SIZE(Factors); static const ui8 CodedFactors[] = { @@ -132,7 +132,7 @@ Y_UNIT_TEST_SUITE(FloatHuffmanTest) { 0x7F, 0x20, 0x1A, 0x81, 0x9A, 0xCA, 0xBF, 0xC8, 0x8D, 0x8D, 0xC2, 0x83, 0x82, 0xA7, 0x2C, 0x28, 0xC8, 0xFE, 0x08, 0xC2, 0x07, 0xC7, 0x27, 0x21, 0xE1, 0xBB, 0x3E, 0xC1, 0x59, 0x68, 0xAA, 0x78, 0xC8, 0x57, 0x5D, 0x60, 0x20, 0xC6, 0x41, 0x42, 0xE8, 0x3A, 0x38, 0xD8, 0x9B, 0xFF, 0xFF, 0xFF, - 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + 0xC4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; static const size_t CodedSize = Y_ARRAY_SIZE(CodedFactors); static const TStringBuf CodedFactorsBuf(reinterpret_cast<const char*>(CodedFactors), CodedSize); @@ -228,7 +228,7 @@ Y_UNIT_TEST_SUITE(FloatHuffmanTest) { "MBD693f07+9+DQQEkIGAgIgPetzN5yEbAGxWpbCNxXK/0JGTKRz2KkIoR7aM"; UNIT_ASSERT_EXCEPTION( fh::Decode(Base64Decode(brokenBase64Encoded)), - yexception); + yexception); } Y_UNIT_TEST(TestDecompressEmpty) { diff --git a/library/cpp/codecs/ut/tls_cache_ut.cpp b/library/cpp/codecs/ut/tls_cache_ut.cpp index 8101af761f..361d41a02e 100644 --- a/library/cpp/codecs/ut/tls_cache_ut.cpp +++ b/library/cpp/codecs/ut/tls_cache_ut.cpp @@ -2,35 +2,35 @@ #include <library/cpp/codecs/tls_cache.h> Y_UNIT_TEST_SUITE(CodecsBufferFactoryTest){ - void AssignToBuffer(TBuffer & buf, TStringBuf val){ + void AssignToBuffer(TBuffer & buf, TStringBuf val){ buf.Assign(val.data(), val.size()); -} +} -TStringBuf AsStringBuf(const TBuffer& b) { - return TStringBuf(b.Data(), b.Size()); -} +TStringBuf AsStringBuf(const TBuffer& b) { + return TStringBuf(b.Data(), b.Size()); +} Y_UNIT_TEST(TestAcquireReleaseReuse) { - NCodecs::TBufferTlsCache factory; - // acquiring the first buffer - auto buf1 = factory.Item(); - AssignToBuffer(buf1.Get(), "Buffer_01"); - { - // acquiring the second buffer - auto buf2 = factory.Item(); - AssignToBuffer(buf2.Get(), "Buffer_02"); + NCodecs::TBufferTlsCache factory; + // acquiring the first buffer + auto buf1 = factory.Item(); + AssignToBuffer(buf1.Get(), "Buffer_01"); + { + // acquiring the second buffer + auto buf2 = factory.Item(); + AssignToBuffer(buf2.Get(), "Buffer_02"); } - // the first buffer should stay intact - UNIT_ASSERT_EQUAL(AsStringBuf(buf1.Get()), "Buffer_01"); - { - // reacquiring the last released buffer - // expecting it zero sized but having the same memory - auto buf2 = factory.Item(); - UNIT_ASSERT_VALUES_EQUAL(buf2.Get().Size(), 0u); - buf2.Get().Resize(TStringBuf("Buffer_02").Size()); - UNIT_ASSERT_EQUAL(AsStringBuf(buf2.Get()), "Buffer_02"); - } - // when the factory dies we should see no leaks -} -} -; + // the first buffer should stay intact + UNIT_ASSERT_EQUAL(AsStringBuf(buf1.Get()), "Buffer_01"); + { + // reacquiring the last released buffer + // expecting it zero sized but having the same memory + auto buf2 = factory.Item(); + UNIT_ASSERT_VALUES_EQUAL(buf2.Get().Size(), 0u); + buf2.Get().Resize(TStringBuf("Buffer_02").Size()); + UNIT_ASSERT_EQUAL(AsStringBuf(buf2.Get()), "Buffer_02"); + } + // when the factory dies we should see no leaks +} +} +; diff --git a/library/cpp/codecs/ya.make b/library/cpp/codecs/ya.make index 7e76fb0c9a..9f7a5b5de2 100644 --- a/library/cpp/codecs/ya.make +++ b/library/cpp/codecs/ya.make @@ -4,7 +4,7 @@ OWNER( g:base velavokr ) - + SRCS( tls_cache.cpp codecs.cpp diff --git a/library/cpp/codecs/zstd_dict_codec.cpp b/library/cpp/codecs/zstd_dict_codec.cpp index c42a2879e6..6aa67abd62 100644 --- a/library/cpp/codecs/zstd_dict_codec.cpp +++ b/library/cpp/codecs/zstd_dict_codec.cpp @@ -28,8 +28,8 @@ namespace NCodecs { TPtrHolder(T* dict) : Ptr(dict) - { - } + { + } T* Get() { return Ptr; @@ -99,7 +99,7 @@ namespace NCodecs { TCCtx ctx{CheckPtr(ZSTD_createCCtx(), __LOCATION__)}; const size_t resSz = CheckSize(ZSTD_compress_usingCDict( ctx.Get(), outbuf.data() + szSz, maxDatSz, rawBeg, rawSz, CDict.Get()), - __LOCATION__); + __LOCATION__); if (resSz < rawSz) { outbuf.Resize(resSz + szSz); @@ -134,13 +134,13 @@ namespace NCodecs { outbuf.Resize(rawSz); memcpy(outbuf.data(), rawBeg, rawSz); } else { - // size_t zSz = ZSTD_getDecompressedSize(rawBeg, rawSz); - // Y_ENSURE_EX(datSz == zSz, TCodecException() << datSz << " != " << zSz); + // size_t zSz = ZSTD_getDecompressedSize(rawBeg, rawSz); + // Y_ENSURE_EX(datSz == zSz, TCodecException() << datSz << " != " << zSz); outbuf.Resize(datSz); TDCtx ctx{CheckPtr(ZSTD_createDCtx(), __LOCATION__)}; CheckSize(ZSTD_decompress_usingDDict( ctx.Get(), outbuf.data(), outbuf.size(), rawBeg, rawSz, DDict.Get()), - __LOCATION__); + __LOCATION__); outbuf.Resize(datSz); } } @@ -206,8 +206,8 @@ namespace NCodecs { template <class T> static T* CheckPtr(T* t, TSourceLocation loc) { - Y_ENSURE_EX(t, TCodecException() << loc << " " - << "unexpected nullptr"); + Y_ENSURE_EX(t, TCodecException() << loc << " " + << "unexpected nullptr"); return t; } @@ -230,8 +230,8 @@ namespace NCodecs { MyTraits.RecommendedSampleSize = TImpl::SampleSize; // same as for solar } - TZStdDictCodec::~TZStdDictCodec() { - } + TZStdDictCodec::~TZStdDictCodec() { + } TString TZStdDictCodec::GetName() const { return TStringBuilder() << MyName() << "-" << Impl->GetCompressionLevel(); diff --git a/library/cpp/codecs/zstd_dict_codec.h b/library/cpp/codecs/zstd_dict_codec.h index 59c1ad6c60..70259989f6 100644 --- a/library/cpp/codecs/zstd_dict_codec.h +++ b/library/cpp/codecs/zstd_dict_codec.h @@ -5,34 +5,34 @@ #include <util/generic/ptr.h> namespace NCodecs { - // benchmarks are here: https://st.yandex-team.ru/SEARCH-1655 + // benchmarks are here: https://st.yandex-team.ru/SEARCH-1655 - class TZStdDictCodec: public ICodec { - class TImpl; - TIntrusivePtr<TImpl> Impl; + class TZStdDictCodec: public ICodec { + class TImpl; + TIntrusivePtr<TImpl> Impl; - public: - explicit TZStdDictCodec(ui32 comprLevel = 1); - ~TZStdDictCodec() override; + public: + explicit TZStdDictCodec(ui32 comprLevel = 1); + ~TZStdDictCodec() override; - static TStringBuf MyName() { - return "zstd08d"; - } + static TStringBuf MyName() { + return "zstd08d"; + } - TString GetName() const override; + TString GetName() const override; - ui8 Encode(TStringBuf in, TBuffer& out) const override; + ui8 Encode(TStringBuf in, TBuffer& out) const override; - void Decode(TStringBuf in, TBuffer& out) const override; + void Decode(TStringBuf in, TBuffer& out) const override; - static TVector<TString> ListCompressionNames(); - static int ParseCompressionName(TStringBuf); + static TVector<TString> ListCompressionNames(); + static int ParseCompressionName(TStringBuf); - protected: - void DoLearn(ISequenceReader& in) override; + protected: + void DoLearn(ISequenceReader& in) override; bool DoTryToLearn(ISequenceReader& in) final; - void Save(IOutputStream* out) const override; - void Load(IInputStream* in) override; - }; + void Save(IOutputStream* out) const override; + void Load(IInputStream* in) override; + }; } diff --git a/library/cpp/colorizer/colors.cpp b/library/cpp/colorizer/colors.cpp index decc5c9847..77cbb53793 100644 --- a/library/cpp/colorizer/colors.cpp +++ b/library/cpp/colorizer/colors.cpp @@ -1,16 +1,16 @@ -#include "colors.h" +#include "colors.h" #include <util/stream/output.h> -#include <util/generic/singleton.h> -#include <util/system/env.h> +#include <util/generic/singleton.h> +#include <util/system/env.h> -#include <cstdlib> - -#if defined(_unix_) -#include <unistd.h> -#endif - -using namespace NColorizer; +#include <cstdlib> + +#if defined(_unix_) +#include <unistd.h> +#endif + +using namespace NColorizer; namespace { constexpr TStringBuf ToStringBufC(NColorizer::EAnsiCode x) { @@ -167,15 +167,15 @@ bool TColors::CalcIsTTY(FILE* file) { } TColors::TColors(FILE* f) - : IsTTY_(true) -{ + : IsTTY_(true) +{ SetIsTTY(CalcIsTTY(f)); -} +} -TColors::TColors(bool ontty) - : IsTTY_(true) -{ - SetIsTTY(ontty); +TColors::TColors(bool ontty) + : IsTTY_(true) +{ + SetIsTTY(ontty); } TStringBuf TColors::Reset() const noexcept { @@ -388,19 +388,19 @@ TStringBuf TColors::LightGrayColor() const noexcept { TStringBuf TColors::DarkGrayColor() const noexcept { return IsTTY() ? "\033[1;30m" : ""; } - + TStringBuf TColors::LightBlueColor() const noexcept { return IsTTY() ? "\033[1;34m" : ""; } - + TStringBuf TColors::LightGreenColor() const noexcept { return IsTTY() ? "\033[1;32m" : ""; } - + TStringBuf TColors::LightCyanColor() const noexcept { return IsTTY() ? "\033[1;36m" : ""; -} - +} + TStringBuf TColors::LightRedColor() const noexcept { return IsTTY() ? "\033[1;31m" : ""; } @@ -415,8 +415,8 @@ TStringBuf TColors::YellowColor() const noexcept { TStringBuf TColors::WhiteColor() const noexcept { return IsTTY() ? "\033[1;37m" : ""; -} - +} + namespace { class TStdErrColors: public TColors { @@ -460,7 +460,7 @@ TColors& NColorizer::AutoColors(IOutputStream& os) { return StdOut(); } return *Singleton<TDisabledColors>(); -} +} size_t NColorizer::TotalAnsiEscapeCodeLen(TStringBuf text) { enum { diff --git a/library/cpp/colorizer/colors.h b/library/cpp/colorizer/colors.h index 474a918994..4535e8ef37 100644 --- a/library/cpp/colorizer/colors.h +++ b/library/cpp/colorizer/colors.h @@ -1,5 +1,5 @@ #pragma once - + #include "fwd.h" #include <util/generic/string.h> @@ -7,7 +7,7 @@ #include <cstdio> -namespace NColorizer { +namespace NColorizer { /** * List of ECMA-48 colors. * @@ -100,7 +100,7 @@ namespace NColorizer { * All color functions return zero-terminated stringbuf. */ class TColors { - public: + public: static bool CalcIsTTY(FILE* file); public: @@ -193,25 +193,25 @@ namespace NColorizer { TStringBuf WhiteColor() const noexcept; inline bool IsTTY() const noexcept { - return IsTTY_; - } + return IsTTY_; + } inline void SetIsTTY(bool value) noexcept { - IsTTY_ = value; - } - + IsTTY_ = value; + } + inline void Enable() noexcept { - SetIsTTY(true); - } - + SetIsTTY(true); + } + inline void Disable() noexcept { - SetIsTTY(false); - } - - private: - bool IsTTY_; + SetIsTTY(false); + } + + private: + bool IsTTY_; }; - + /// Singletone `TColors` instances for stderr/stdout. TColors& StdErr(); TColors& StdOut(); diff --git a/library/cpp/colorizer/fwd.h b/library/cpp/colorizer/fwd.h index d71efdc053..5ab514ad9d 100644 --- a/library/cpp/colorizer/fwd.h +++ b/library/cpp/colorizer/fwd.h @@ -8,4 +8,4 @@ namespace NColorizer { TColors& StdErr(); TColors& StdOut(); TColors& AutoColors(IOutputStream&); -} +} diff --git a/library/cpp/colorizer/output.cpp b/library/cpp/colorizer/output.cpp index 66a5626675..266a8adf96 100644 --- a/library/cpp/colorizer/output.cpp +++ b/library/cpp/colorizer/output.cpp @@ -1,10 +1,10 @@ -#include "output.h" - -#include <util/stream/output.h> - -using namespace NColorizer; - -template <> +#include "output.h" + +#include <util/stream/output.h> + +using namespace NColorizer; + +template <> void Out<TColorHandle>(IOutputStream& o, const TColorHandle& h) { - o << (*(h.C).*h.F)(); -} + o << (*(h.C).*h.F)(); +} diff --git a/library/cpp/colorizer/output.h b/library/cpp/colorizer/output.h index 99afbd3427..b33e5f49f0 100644 --- a/library/cpp/colorizer/output.h +++ b/library/cpp/colorizer/output.h @@ -1,34 +1,34 @@ -#pragma once - -#include "colors.h" - +#pragma once + +#include "colors.h" + // Note: this is an old interface for printing colors to stream. // Consider printing elements of `EAnsiCode` directly. -namespace NColorizer { - typedef TStringBuf (TColors::*TColorFunc)() const; - - struct TColorHandle { - const TColors* C; - TColorFunc F; - +namespace NColorizer { + typedef TStringBuf (TColors::*TColorFunc)() const; + + struct TColorHandle { + const TColors* C; + TColorFunc F; + inline TColorHandle(const TColors* c, TColorFunc f) noexcept - : C(c) - , F(f) - { - } - }; - -#define DEF(X) \ - static inline TColorHandle X() noexcept { \ - return TColorHandle(&StdErr(), &TColors::X##Color); \ - } - - DEF(Old) + : C(c) + , F(f) + { + } + }; + +#define DEF(X) \ + static inline TColorHandle X() noexcept { \ + return TColorHandle(&StdErr(), &TColors::X##Color); \ + } + + DEF(Old) DEF(Black) - DEF(Green) + DEF(Green) DEF(Cyan) - DEF(Red) + DEF(Red) DEF(Purple) DEF(Brown) DEF(LightGray) @@ -38,8 +38,8 @@ namespace NColorizer { DEF(LightCyan) DEF(LightRed) DEF(LightPurple) - DEF(Yellow) - DEF(White) - -#undef DEF -} + DEF(Yellow) + DEF(White) + +#undef DEF +} diff --git a/library/cpp/colorizer/ya.make b/library/cpp/colorizer/ya.make index 141edb2797..380a2f9855 100644 --- a/library/cpp/colorizer/ya.make +++ b/library/cpp/colorizer/ya.make @@ -1,10 +1,10 @@ -LIBRARY() - -OWNER(pg) - -SRCS( - colors.cpp - output.cpp -) - -END() +LIBRARY() + +OWNER(pg) + +SRCS( + colors.cpp + output.cpp +) + +END() diff --git a/library/cpp/compproto/bit.h b/library/cpp/compproto/bit.h index 6a421b65f7..7d2800843f 100644 --- a/library/cpp/compproto/bit.h +++ b/library/cpp/compproto/bit.h @@ -3,346 +3,346 @@ #include <util/generic/array_ref.h> #include <util/generic/vector.h> #include <util/stream/output.h> -#include <util/stream/input.h> +#include <util/stream/input.h> #include "huff.h" #include "compressor.h" -#include "metainfo.h" +#include "metainfo.h" namespace NCompProto { - struct TBitBuffer { - TVector<ui8> Out; - ui64 Position; - ui64 Size; - ui8 Counter; - TBitBuffer() { - Clear(); - } - - static ui64 Read(const ui8* out, ui64 position, size_t size) { - const ui8* dst = out + (size_t(position >> 3)); - ui64 outCode = *reinterpret_cast<const ui64*>(dst); - ui8 shift = position & 7; - return ((1ULL << size) - 1) & (outCode >> shift); - } - - ui64 Read(ui64 position, size_t size) { - return Read(&Out[0], position, size); - } - - void Code(ui64 value, size_t size) { - if (++Counter == 0) { - Junk(257 * 64); - } - Position = Code(value, size, Position); - } - ui64 Code(ui64 value, size_t size, ui64 position) { - ui8* dst = &Out[size_t(position >> 3)]; - ui64& outCode = *(ui64*)dst; - ui8 shift = position & 7; - ui64 mask = ((1ULL << size) - 1) << shift; - outCode = ((value << shift) & mask) | (outCode & ~mask); - return position + size; - } - void Junk(size_t junk = 1024) { - size_t need = size_t(Position >> 3); - if (Out.size() * 8 < Position + junk) { - Out.resize(((need + junk) * 3) / 2); - Size = Out.size(); - } - } - void Insert(ui64 value, ui64 position, size_t size) { - ui64 newVal = Read(position, 56); - position = Code(value, size, position); - value = newVal; - - while (position < Position + 64) { - newVal = Read(position + 56 - size, 56); - position = Code(value, 56, position); - value = newVal; - } - - Position += size; - } - - size_t ByteLength() const { - return (Position + 7) / 8; - } - - void ResetPosition() { - Position = 0; - Size = 0; - } - - void Clear() { - Counter = 0; - Position = 0; - Size = 0; - Out.clear(); - Junk(); - } + struct TBitBuffer { + TVector<ui8> Out; + ui64 Position; + ui64 Size; + ui8 Counter; + TBitBuffer() { + Clear(); + } + + static ui64 Read(const ui8* out, ui64 position, size_t size) { + const ui8* dst = out + (size_t(position >> 3)); + ui64 outCode = *reinterpret_cast<const ui64*>(dst); + ui8 shift = position & 7; + return ((1ULL << size) - 1) & (outCode >> shift); + } + + ui64 Read(ui64 position, size_t size) { + return Read(&Out[0], position, size); + } + + void Code(ui64 value, size_t size) { + if (++Counter == 0) { + Junk(257 * 64); + } + Position = Code(value, size, Position); + } + ui64 Code(ui64 value, size_t size, ui64 position) { + ui8* dst = &Out[size_t(position >> 3)]; + ui64& outCode = *(ui64*)dst; + ui8 shift = position & 7; + ui64 mask = ((1ULL << size) - 1) << shift; + outCode = ((value << shift) & mask) | (outCode & ~mask); + return position + size; + } + void Junk(size_t junk = 1024) { + size_t need = size_t(Position >> 3); + if (Out.size() * 8 < Position + junk) { + Out.resize(((need + junk) * 3) / 2); + Size = Out.size(); + } + } + void Insert(ui64 value, ui64 position, size_t size) { + ui64 newVal = Read(position, 56); + position = Code(value, size, position); + value = newVal; + + while (position < Position + 64) { + newVal = Read(position + 56 - size, 56); + position = Code(value, 56, position); + value = newVal; + } + + Position += size; + } + + size_t ByteLength() const { + return (Position + 7) / 8; + } + + void ResetPosition() { + Position = 0; + Size = 0; + } + + void Clear() { + Counter = 0; + Position = 0; + Size = 0; + Out.clear(); + Junk(); + } TArrayRef<const char> AsDataRegion() const { return TArrayRef<const char>(reinterpret_cast<const char*>(Out.data()), ByteLength()); - } - }; - - struct THuff { - TCoder Coder; - ui64 Position; - THuff() { - Position = (ui64)(-1); - } - void Load(IInputStream& stream) { - TString name; - Coder.InitDefault(); - Coder.Entries.clear(); - while (1) { - stream >> name; - if (name == "end") - return; - else if (name == "entry") { - TCoderEntry entry; - ui32 value; - stream >> value; - entry.MinValue = value; - stream >> value; - entry.Prefix = value; - stream >> value; - entry.PrefixBits = value; - stream >> value; - entry.AllBits = value; - Coder.Entries.push_back(entry); - } - } - } - - void Save(IOutputStream& stream, TString offset) { - TString step = " "; - for (size_t i = 0; i < Coder.Entries.size(); ++i) { - stream << offset << step << "entry "; - stream << (ui32)Coder.Entries[i].MinValue << " "; - stream << (ui32)Coder.Entries[i].Prefix << " "; - stream << (ui32)Coder.Entries[i].PrefixBits << " "; - stream << (ui32)Coder.Entries[i].AllBits << " "; - stream << Endl; - } - stream << offset << "end" << Endl; - } - - void BeginElement(TBitBuffer& out) { - Position = out.Position; - } - void Add(ui32 value, TBitBuffer& out) { - size_t val = 0; - ui64 code = Coder.Code(value, val); - out.Code(code, val); - } - void AddDelayed(ui32 value, TBitBuffer& out) { - size_t val = 0; - ui64 code = Coder.Code(value, val); - if (Position == (ui64)(-1)) { - ythrow yexception() << "Position == (ui64)(-1)"; - } - out.Insert(code, Position, val); - out.Junk(); - } - }; - - struct THist { - TAccum Accum; - THist() { - } - - void Load(IInputStream& stream) { - TString name; - while (1) { - stream >> name; - if (name == "end") - return; + } + }; + + struct THuff { + TCoder Coder; + ui64 Position; + THuff() { + Position = (ui64)(-1); + } + void Load(IInputStream& stream) { + TString name; + Coder.InitDefault(); + Coder.Entries.clear(); + while (1) { + stream >> name; + if (name == "end") + return; + else if (name == "entry") { + TCoderEntry entry; + ui32 value; + stream >> value; + entry.MinValue = value; + stream >> value; + entry.Prefix = value; + stream >> value; + entry.PrefixBits = value; + stream >> value; + entry.AllBits = value; + Coder.Entries.push_back(entry); + } + } + } + + void Save(IOutputStream& stream, TString offset) { + TString step = " "; + for (size_t i = 0; i < Coder.Entries.size(); ++i) { + stream << offset << step << "entry "; + stream << (ui32)Coder.Entries[i].MinValue << " "; + stream << (ui32)Coder.Entries[i].Prefix << " "; + stream << (ui32)Coder.Entries[i].PrefixBits << " "; + stream << (ui32)Coder.Entries[i].AllBits << " "; + stream << Endl; } - } - // TODO: why not const TString& ??? - void Save(IOutputStream& /*stream*/, TString /*offset*/) { - } - - void Add(ui32 value, TEmpty& /*empty*/) { - Accum.Add(value); - } - void AddDelayed(ui32 value, TEmpty& /*empty*/) { - Accum.Add(value); - } - void BeginElement(TEmpty& /*empty*/) { - } - }; - - struct THistToHuff { - static THistToHuff Instance() { - return THistToHuff(); - } - TEmpty Build() const { - return TEmpty(); - } - void Build(THuff& info, const THist& hist) const { - Analyze(hist.Accum, info.Coder.Entries); - info.Coder.Normalize(); - } - }; - - struct IDecompressor { - // sequentially decompresses whole structure according to metainfo, starts at position offset - virtual void Decompress(const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset) = 0; - // decompresses one record of outer repeated structure starting at position offset - virtual void DecompressOne(const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset, ui32 prevIndex = -1) = 0; - virtual ~IDecompressor() = default; - }; - - template <class X> - struct TMetaIterator: public IDecompressor { - X Self; - TMetaIterator() { - Self.Parent = this; - } - - private: - inline void DecompressSingle(ui32 repeatedIndex, const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset) { - Self.BeginElement(repeatedIndex); - ui32 mask = table->Mask.Decompress(codes, offset); - size_t index = 0; - ui32 scalarMask = table->ScalarMask; - while (mask || scalarMask) { - if (mask & 1) { - if (scalarMask & 1) { - ui32 val = table->Scalar[index].Decompress(codes, offset); - Self.SetScalar(index, val); - } else { - Self.GetDecompressor(index).Decompress(table->Repeated[index].Get(), codes, offset); - } - } else if ((scalarMask & 1) && table->Default[index].Type == TScalarDefaultValue::Fixed) { - Self.SetScalar(index, table->Default[index].Value); - } - scalarMask = scalarMask >> 1; - mask = mask >> 1; - ++index; - } - Self.EndElement(); - } - - inline void DecompressSingleScalarsOnly(ui32 repeatedIndex, const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset) { - Self.BeginElement(repeatedIndex); - ui32 mask = table->Mask.Decompress(codes, offset); - ui32 scalarMask = table->ScalarMask; - size_t index = 0; - while (scalarMask) { - if (mask & 1) { + stream << offset << "end" << Endl; + } + + void BeginElement(TBitBuffer& out) { + Position = out.Position; + } + void Add(ui32 value, TBitBuffer& out) { + size_t val = 0; + ui64 code = Coder.Code(value, val); + out.Code(code, val); + } + void AddDelayed(ui32 value, TBitBuffer& out) { + size_t val = 0; + ui64 code = Coder.Code(value, val); + if (Position == (ui64)(-1)) { + ythrow yexception() << "Position == (ui64)(-1)"; + } + out.Insert(code, Position, val); + out.Junk(); + } + }; + + struct THist { + TAccum Accum; + THist() { + } + + void Load(IInputStream& stream) { + TString name; + while (1) { + stream >> name; + if (name == "end") + return; + } + } + // TODO: why not const TString& ??? + void Save(IOutputStream& /*stream*/, TString /*offset*/) { + } + + void Add(ui32 value, TEmpty& /*empty*/) { + Accum.Add(value); + } + void AddDelayed(ui32 value, TEmpty& /*empty*/) { + Accum.Add(value); + } + void BeginElement(TEmpty& /*empty*/) { + } + }; + + struct THistToHuff { + static THistToHuff Instance() { + return THistToHuff(); + } + TEmpty Build() const { + return TEmpty(); + } + void Build(THuff& info, const THist& hist) const { + Analyze(hist.Accum, info.Coder.Entries); + info.Coder.Normalize(); + } + }; + + struct IDecompressor { + // sequentially decompresses whole structure according to metainfo, starts at position offset + virtual void Decompress(const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset) = 0; + // decompresses one record of outer repeated structure starting at position offset + virtual void DecompressOne(const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset, ui32 prevIndex = -1) = 0; + virtual ~IDecompressor() = default; + }; + + template <class X> + struct TMetaIterator: public IDecompressor { + X Self; + TMetaIterator() { + Self.Parent = this; + } + + private: + inline void DecompressSingle(ui32 repeatedIndex, const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset) { + Self.BeginElement(repeatedIndex); + ui32 mask = table->Mask.Decompress(codes, offset); + size_t index = 0; + ui32 scalarMask = table->ScalarMask; + while (mask || scalarMask) { + if (mask & 1) { + if (scalarMask & 1) { + ui32 val = table->Scalar[index].Decompress(codes, offset); + Self.SetScalar(index, val); + } else { + Self.GetDecompressor(index).Decompress(table->Repeated[index].Get(), codes, offset); + } + } else if ((scalarMask & 1) && table->Default[index].Type == TScalarDefaultValue::Fixed) { + Self.SetScalar(index, table->Default[index].Value); + } + scalarMask = scalarMask >> 1; + mask = mask >> 1; + ++index; + } + Self.EndElement(); + } + + inline void DecompressSingleScalarsOnly(ui32 repeatedIndex, const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset) { + Self.BeginElement(repeatedIndex); + ui32 mask = table->Mask.Decompress(codes, offset); + ui32 scalarMask = table->ScalarMask; + size_t index = 0; + while (scalarMask) { + if (mask & 1) { ui32 val = table->Scalar[index].Decompress(codes, offset); Self.SetScalar(index, val); - } else if (table->Default[index].Type == TScalarDefaultValue::Fixed) { - Self.SetScalar(index, table->Default[index].Value); + } else if (table->Default[index].Type == TScalarDefaultValue::Fixed) { + Self.SetScalar(index, table->Default[index].Value); } - mask = mask >> 1; - scalarMask = scalarMask >> 1; - ++index; + mask = mask >> 1; + scalarMask = scalarMask >> 1; + ++index; } - Self.EndElement(); - } - - public: - void Decompress(const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset) override { - ui64 locOffset = offset; - ui32 count = table->Count.Decompress(codes, locOffset); - ui32 repeatedIndex = (ui32)(-1); - Self.BeginSelf(count, table->Id); - if (table->RepeatedMask) { - for (ui32 i = 0; i < count; ++i) { - repeatedIndex += table->Index.Decompress(codes, locOffset); - DecompressSingle(repeatedIndex, table, codes, locOffset); - } - } else { - for (ui32 i = 0; i < count; ++i) { - repeatedIndex += table->Index.Decompress(codes, locOffset); - DecompressSingleScalarsOnly(repeatedIndex, table, codes, locOffset); - } + Self.EndElement(); + } + + public: + void Decompress(const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset) override { + ui64 locOffset = offset; + ui32 count = table->Count.Decompress(codes, locOffset); + ui32 repeatedIndex = (ui32)(-1); + Self.BeginSelf(count, table->Id); + if (table->RepeatedMask) { + for (ui32 i = 0; i < count; ++i) { + repeatedIndex += table->Index.Decompress(codes, locOffset); + DecompressSingle(repeatedIndex, table, codes, locOffset); + } + } else { + for (ui32 i = 0; i < count; ++i) { + repeatedIndex += table->Index.Decompress(codes, locOffset); + DecompressSingleScalarsOnly(repeatedIndex, table, codes, locOffset); + } } - offset = locOffset; - Self.EndSelf(); - } - - // XXX: iterator needed? - // - // Following two functions serves the purpose of decompressing outer repeated-structure(such structure is mandatory now). - // They can work for inner repeated structures too, if you supply correct table, codes and offset parameters. - void DecompressOne(const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset, ui32 prevIndex = -1) override { - table->Index.Decompress(codes, offset); - DecompressSingle(prevIndex, table, codes, offset); - } - - ui32 DecompressCount(const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset) { - return table->Count.Decompress(codes, offset); - } - }; - - template <class X> - struct TParentHold { - TMetaIterator<X>* Parent; - TParentHold() - : Parent(nullptr) - { - } - }; - - struct TEmptyDecompressor: public TParentHold<TEmptyDecompressor> { - void BeginSelf(ui32 /*count*/, ui32 /*id*/) { - } - void EndSelf() { - } - void BeginElement(ui32 /*element*/) { - } - void EndElement() { - } - void SetScalar(size_t /*index*/, ui32 /*val*/) { - } - TMetaIterator<TEmptyDecompressor>& GetDecompressor(size_t index) { - Y_UNUSED(index); - return *Parent; - } - }; - - inline TMetaIterator<TEmptyDecompressor>& GetEmptyDecompressor() { - static TMetaIterator<TEmptyDecompressor> empty; - return empty; + offset = locOffset; + Self.EndSelf(); + } + + // XXX: iterator needed? + // + // Following two functions serves the purpose of decompressing outer repeated-structure(such structure is mandatory now). + // They can work for inner repeated structures too, if you supply correct table, codes and offset parameters. + void DecompressOne(const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset, ui32 prevIndex = -1) override { + table->Index.Decompress(codes, offset); + DecompressSingle(prevIndex, table, codes, offset); + } + + ui32 DecompressCount(const TMetaInfo<TTable>* table, const ui8* codes, ui64& offset) { + return table->Count.Decompress(codes, offset); + } + }; + + template <class X> + struct TParentHold { + TMetaIterator<X>* Parent; + TParentHold() + : Parent(nullptr) + { + } + }; + + struct TEmptyDecompressor: public TParentHold<TEmptyDecompressor> { + void BeginSelf(ui32 /*count*/, ui32 /*id*/) { + } + void EndSelf() { + } + void BeginElement(ui32 /*element*/) { + } + void EndElement() { + } + void SetScalar(size_t /*index*/, ui32 /*val*/) { + } + TMetaIterator<TEmptyDecompressor>& GetDecompressor(size_t index) { + Y_UNUSED(index); + return *Parent; + } + }; + + inline TMetaIterator<TEmptyDecompressor>& GetEmptyDecompressor() { + static TMetaIterator<TEmptyDecompressor> empty; + return empty; } - struct THuffToTable { - static THuffToTable Instance() { - return THuffToTable(); - } - void Build(TTable& table, const THuff& huff) const { - // can happen if a field was never serialized across whole structure - if (huff.Coder.Entries.empty()) - return; - for (ui32 i = 0; i < 64; ++i) { - const TCoderEntry& entry = huff.Coder.GetEntry(i, table.Id[i]); - table.CodeBase[i] = entry.MinValue; - table.PrefLength[i] = entry.PrefixBits; - table.CodeMask[i] = (1ULL << (entry.AllBits - entry.PrefixBits)) - 1ULL; - table.Length[i] = entry.AllBits; - } - } - }; - - struct THuffToTableWithDecompressor: private THuffToTable { - TSimpleSharedPtr<IDecompressor> Decompressor; - THuffToTableWithDecompressor(TSimpleSharedPtr<IDecompressor> decompressor) - : Decompressor(decompressor) - { - } - TSimpleSharedPtr<IDecompressor> Build() const { - return Decompressor; - } - void Build(TTable& table, const THuff& huff) const { - THuffToTable::Build(table, huff); - } - }; - -} + struct THuffToTable { + static THuffToTable Instance() { + return THuffToTable(); + } + void Build(TTable& table, const THuff& huff) const { + // can happen if a field was never serialized across whole structure + if (huff.Coder.Entries.empty()) + return; + for (ui32 i = 0; i < 64; ++i) { + const TCoderEntry& entry = huff.Coder.GetEntry(i, table.Id[i]); + table.CodeBase[i] = entry.MinValue; + table.PrefLength[i] = entry.PrefixBits; + table.CodeMask[i] = (1ULL << (entry.AllBits - entry.PrefixBits)) - 1ULL; + table.Length[i] = entry.AllBits; + } + } + }; + + struct THuffToTableWithDecompressor: private THuffToTable { + TSimpleSharedPtr<IDecompressor> Decompressor; + THuffToTableWithDecompressor(TSimpleSharedPtr<IDecompressor> decompressor) + : Decompressor(decompressor) + { + } + TSimpleSharedPtr<IDecompressor> Build() const { + return Decompressor; + } + void Build(TTable& table, const THuff& huff) const { + THuffToTable::Build(table, huff); + } + }; + +} diff --git a/library/cpp/compproto/compproto_ut.cpp b/library/cpp/compproto/compproto_ut.cpp index 9393be967a..e0cfcfaf6b 100644 --- a/library/cpp/compproto/compproto_ut.cpp +++ b/library/cpp/compproto/compproto_ut.cpp @@ -32,8 +32,8 @@ struct TTestParams { ui32 ValueArraySize; }; -template <typename X> -void TestSaveLoadMeta(NCompProto::TMetaInfo<X>& src) { +template <typename X> +void TestSaveLoadMeta(NCompProto::TMetaInfo<X>& src) { TStringStream ss; src.Save(ss); TString data = ss.Str(); @@ -50,7 +50,7 @@ void TestWithParams(const TString& metainfo, const ECompMode mode, const TTestPa TStringInput stream(metainfo); - THolder<TMetaInfo<THuff>> meta; + THolder<TMetaInfo<THuff>> meta; if (mode == CM_TWOPASS) { TMetaInfo<THist> hist(stream); TEmpty empty; @@ -70,19 +70,19 @@ void TestWithParams(const TString& metainfo, const ECompMode mode, const TTestPa // verify that no memory read beyond buffer occurs const size_t byteSize = buffer.ByteLength(); - const size_t PAGESIZEX = 4096; - const size_t busyPages = (byteSize + (PAGESIZEX - 1)) / PAGESIZEX; + const size_t PAGESIZEX = 4096; + const size_t busyPages = (byteSize + (PAGESIZEX - 1)) / PAGESIZEX; const size_t allPages = busyPages + 1; - const size_t allocSize = (allPages + 1) * PAGESIZEX; + const size_t allocSize = (allPages + 1) * PAGESIZEX; TVector<ui8> readBuffer(allocSize); ui8* start = &readBuffer[0]; - ui8* pageStart = reinterpret_cast<ui8*>((size_t(start) + PAGESIZEX) & ~(PAGESIZEX - 1)); + ui8* pageStart = reinterpret_cast<ui8*>((size_t(start) + PAGESIZEX) & ~(PAGESIZEX - 1)); // XX DATA DATA DATA DATA PROT // | | | | | pages // calculate dataStart so that data ends exactly at the page end - ui8* dataStart = pageStart + busyPages * PAGESIZEX - byteSize; - ui8* dataEnd = pageStart + busyPages * PAGESIZEX; - ProtectMemory(dataEnd, PAGESIZEX, PM_NONE); + ui8* dataStart = pageStart + busyPages * PAGESIZEX - byteSize; + ui8* dataEnd = pageStart + busyPages * PAGESIZEX; + ProtectMemory(dataEnd, PAGESIZEX, PM_NONE); // memory copying should be performed without any problems memcpy(dataStart, buffer.Out.data(), byteSize); @@ -93,7 +93,7 @@ void TestWithParams(const TString& metainfo, const ECompMode mode, const TTestPa const ui64 decodedSize = position; UNIT_ASSERT_EQUAL(codedSize, decodedSize); // unprotect memory - ProtectMemory(dataEnd, PAGESIZEX, PM_READ | PM_WRITE | PM_EXEC); + ProtectMemory(dataEnd, PAGESIZEX, PM_READ | PM_WRITE | PM_EXEC); } template <typename TDecompressor, template <typename, typename> class TSerialize> @@ -112,7 +112,7 @@ Y_UNIT_TEST_SUITE(CompProtoTestBasic) { using namespace NCompProto; const TString metainfo = - "\n\ + "\n\ repeated data id 0\n\ scalar clicks id 0 default const 0\n\ scalar shows id 1 default const 0\n\ @@ -137,9 +137,9 @@ Y_UNIT_TEST_SUITE(CompProtoTestBasic) { TVector<TData> data; - template <class TMeta, class TFunctor> + template <class TMeta, class TFunctor> struct TSerialize { - static void Serialize(TMetaInfo<TMeta>& meta, TFunctor& functor, const TTestParams& params) { + static void Serialize(TMetaInfo<TMeta>& meta, TFunctor& functor, const TTestParams& params) { FlushPseudoRandom(); meta.BeginSelf(functor); data.clear(); @@ -153,7 +153,7 @@ Y_UNIT_TEST_SUITE(CompProtoTestBasic) { meta.SetScalar(0, data[i].Clicks, functor); meta.SetScalar(1, data[i].Shows, functor); - TMetaInfo<TMeta>& regClicks = meta.BeginRepeated(2, functor); + TMetaInfo<TMeta>& regClicks = meta.BeginRepeated(2, functor); for (ui32 j = 0; j < PseudoRandom(200); j += 1 + PseudoRandom(10)) { regClicks.BeginElement(j, functor); TRegInfo& r = data[i].RegClicks[j]; @@ -173,10 +173,10 @@ Y_UNIT_TEST_SUITE(CompProtoTestBasic) { } }; - struct TMultiDecompressor: public TParentHold<TMultiDecompressor> { - struct TRegClicks: public TParentHold<TRegClicks> { - const TData* Data; - const TRegInfo* Elem; + struct TMultiDecompressor: public TParentHold<TMultiDecompressor> { + struct TRegClicks: public TParentHold<TRegClicks> { + const TData* Data; + const TRegInfo* Elem; TRegClicks() : Data(nullptr) , Elem(nullptr) @@ -201,13 +201,13 @@ Y_UNIT_TEST_SUITE(CompProtoTestBasic) { if (index == 1) UNIT_ASSERT_EQUAL(val, Elem->Shows); } - IDecompressor& GetDecompressor(size_t) { + IDecompressor& GetDecompressor(size_t) { UNIT_ASSERT(0); return GetEmptyDecompressor(); } }; - const TData* Elem; + const TData* Elem; TMetaIterator<TRegClicks> RegClicks; void BeginSelf(ui32 /*count*/, ui32 /*id*/) { } @@ -227,7 +227,7 @@ Y_UNIT_TEST_SUITE(CompProtoTestBasic) { if (index == 31) UNIT_ASSERT_EQUAL(val, Elem->Extra); } - IDecompressor& GetDecompressor(size_t index) { + IDecompressor& GetDecompressor(size_t index) { if (index == 2) { RegClicks.Self.Data = Elem; return RegClicks; @@ -241,19 +241,19 @@ Y_UNIT_TEST_SUITE(CompProtoTestBasic) { } }; - struct TVerifyingDecompressor: public TParentHold<TVerifyingDecompressor> { + struct TVerifyingDecompressor: public TParentHold<TVerifyingDecompressor> { enum EState { - Startstop, - OutDataElem, - InDataElem, - InRegClicks, + Startstop, + OutDataElem, + InDataElem, + InRegClicks, }; EState State; ui32 DataInd; TMap<ui32, TRegInfo>::iterator RegIter; - TMetaIterator<TVerifyingDecompressor>& GetDecompressor(size_t index) { + TMetaIterator<TVerifyingDecompressor>& GetDecompressor(size_t index) { Y_UNUSED(index); return *Parent; } @@ -374,7 +374,7 @@ Y_UNIT_TEST_SUITE(CompProtoTestBasic) { Y_UNIT_TEST_SUITE(CompProtoTestExtended) { using namespace NCompProto; const TString metainfo = - "\n\ + "\n\ repeated data id 0\n\ repeated second id 3\n\ scalar inner2 id 0 default const 0\n\ @@ -385,21 +385,21 @@ Y_UNIT_TEST_SUITE(CompProtoTestExtended) { end\n"; TVector<std::pair<TVector<ui32>, TVector<ui32>>> data; - template <class TMeta, class TFunctor> + template <class TMeta, class TFunctor> struct TSerialize { - static void Serialize(TMetaInfo<TMeta>& meta, TFunctor& functor, const TTestParams& params) { + static void Serialize(TMetaInfo<TMeta>& meta, TFunctor& functor, const TTestParams& params) { FlushPseudoRandom(); meta.BeginSelf(functor); data.clear(); data.resize(params.DataSize); for (size_t i = 0; i < params.DataSize; ++i) { meta.BeginElement(i, functor); - TMetaInfo<TMeta>& first = meta.BeginRepeated(2, functor); + TMetaInfo<TMeta>& first = meta.BeginRepeated(2, functor); data[i].first.resize(params.ValueArraySize); for (ui32 j = 0; j < params.ValueArraySize; j++) { first.BeginElement(j, functor); - ui32 val = PseudoRandom(42 * 42 * 42); + ui32 val = PseudoRandom(42 * 42 * 42); first.SetScalar(0, val, functor); data[i].first[j] = val; @@ -407,12 +407,12 @@ Y_UNIT_TEST_SUITE(CompProtoTestExtended) { } first.EndRepeated(functor); - TMetaInfo<TMeta>& second = meta.BeginRepeated(3, functor); + TMetaInfo<TMeta>& second = meta.BeginRepeated(3, functor); data[i].second.resize(params.ValueArraySize); for (ui32 j = 0; j < params.ValueArraySize; j++) { second.BeginElement(j, functor); - ui32 val = PseudoRandom(42 * 42 * 42); + ui32 val = PseudoRandom(42 * 42 * 42); second.SetScalar(0, val, functor); data[i].second[j] = val; @@ -425,14 +425,14 @@ Y_UNIT_TEST_SUITE(CompProtoTestExtended) { } }; - struct TVerifyingDecompressor: public TParentHold<TVerifyingDecompressor> { + struct TVerifyingDecompressor: public TParentHold<TVerifyingDecompressor> { enum EState { - Startstop, - OutDataElem, - InDataElemBeforeSecond, - InDataElemSecond, - InFirst, - InSecond, + Startstop, + OutDataElem, + InDataElemBeforeSecond, + InDataElemSecond, + InFirst, + InSecond, }; EState State; @@ -446,7 +446,7 @@ Y_UNIT_TEST_SUITE(CompProtoTestExtended) { { } - TMetaIterator<TVerifyingDecompressor>& GetDecompressor(size_t index) { + TMetaIterator<TVerifyingDecompressor>& GetDecompressor(size_t index) { Y_UNUSED(index); return *Parent; } diff --git a/library/cpp/compproto/compressor.h b/library/cpp/compproto/compressor.h index 14b335e13c..0e3a27afd0 100644 --- a/library/cpp/compproto/compressor.h +++ b/library/cpp/compproto/compressor.h @@ -3,72 +3,72 @@ #include <util/system/defaults.h> namespace NCompProto { - struct TEmpty { - }; + struct TEmpty { + }; - struct TTable { - TTable() { - for (size_t i = 0; i < 64; ++i) { - CodeBase[i] = 0; - CodeMask[i] = 0; - Length[i] = 0; - PrefLength[i] = 0; - Id[i] = 0; - } + struct TTable { + TTable() { + for (size_t i = 0; i < 64; ++i) { + CodeBase[i] = 0; + CodeMask[i] = 0; + Length[i] = 0; + PrefLength[i] = 0; + Id[i] = 0; + } } - ui32 CodeBase[64]; - ui32 CodeMask[64]; - ui8 Length[64]; - ui8 PrefLength[64]; - ui8 Id[64]; - enum { - PAGE_BOUND = 4096, + ui32 CodeBase[64]; + ui32 CodeMask[64]; + ui8 Length[64]; + ui8 PrefLength[64]; + ui8 Id[64]; + enum { + PAGE_BOUND = 4096, #ifdef WITH_VALGRIND - SAFE_MODE = 1, + SAFE_MODE = 1, #else -#if defined(__has_feature) -#if __has_feature(address_sanitizer) +#if defined(__has_feature) +#if __has_feature(address_sanitizer) SAFE_MODE = 1, -#else +#else SAFE_MODE = 0, #endif -#else - SAFE_MODE = 0, -#endif -#endif - }; - ui32 inline Decompress(const ui8* codes, ui64& offset) const { - codes += (offset >> 3); - size_t pageOff = size_t(codes) % PAGE_BOUND; - size_t readOff = offset & 7; - if (pageOff > PAGE_BOUND - 8 || SAFE_MODE) { - size_t off = 8; - ui64 res = codes[0]; +#else + SAFE_MODE = 0, +#endif +#endif + }; + ui32 inline Decompress(const ui8* codes, ui64& offset) const { + codes += (offset >> 3); + size_t pageOff = size_t(codes) % PAGE_BOUND; + size_t readOff = offset & 7; + if (pageOff > PAGE_BOUND - 8 || SAFE_MODE) { + size_t off = 8; + ui64 res = codes[0]; ++codes; - ui64 indexCur = ((res + 0x0000) >> readOff) & 63; - ui64 indexAlt = ((res + 0xff00) >> readOff) & 63; - if (Id[indexCur] != Id[indexAlt]) { - res += (ui64(codes[0]) << off); - ++codes; - off += 8; - indexCur = (res >> readOff) & 63; - } - ui64 index = indexCur; - ui64 length = Length[index]; - while (off < readOff + length) { - res += (ui64(codes[0]) << off); - ++codes; - off += 8; - } - offset += length; - ui64 code = res >> readOff; - return (((ui32)(code >> PrefLength[index])) & CodeMask[index]) + CodeBase[index]; + ui64 indexCur = ((res + 0x0000) >> readOff) & 63; + ui64 indexAlt = ((res + 0xff00) >> readOff) & 63; + if (Id[indexCur] != Id[indexAlt]) { + res += (ui64(codes[0]) << off); + ++codes; + off += 8; + indexCur = (res >> readOff) & 63; + } + ui64 index = indexCur; + ui64 length = Length[index]; + while (off < readOff + length) { + res += (ui64(codes[0]) << off); + ++codes; + off += 8; + } + offset += length; + ui64 code = res >> readOff; + return (((ui32)(code >> PrefLength[index])) & CodeMask[index]) + CodeBase[index]; } - ui64 code = ((const ui64*)(codes))[0] >> readOff; - ui64 index = code & 63; - offset += Length[index]; + ui64 code = ((const ui64*)(codes))[0] >> readOff; + ui64 index = code & 63; + offset += Length[index]; return (((ui32)(code >> PrefLength[index])) & CodeMask[index]) + CodeBase[index]; } - }; + }; -} +} diff --git a/library/cpp/compproto/huff.h b/library/cpp/compproto/huff.h index fa5c139189..a27c014bbb 100644 --- a/library/cpp/compproto/huff.h +++ b/library/cpp/compproto/huff.h @@ -7,396 +7,396 @@ #include <util/generic/algorithm.h> #include <utility> -#include <queue> +#include <queue> #include "compressor.h" namespace NCompProto { - template <size_t CacheSize, typename TEntry> - struct TCache { - ui32 CacheKey[CacheSize]; - TEntry CacheVal[CacheSize]; - size_t Hits; - size_t Misses; - ui32 Hash(ui32 key) { - return key % CacheSize; - } - TCache() { - Hits = 0; - Misses = 0; - Clear(); - } - void Clear() { - for (size_t i = 0; i < CacheSize; ++i) { - ui32 j = 0; - for (; Hash(j) == i; ++j) - ; - CacheKey[i] = j; - } - } - }; - - struct TCode { - i64 Probability; - ui32 Start; - ui32 Bits; - ui32 Prefix; - ui32 PrefLength; - TCode(i64 probability = 0, ui32 start = 0, ui32 bits = 0) - : Probability(probability) - , Start(start) - , Bits(bits) - { - } - - bool operator<(const TCode& code) const { - return Probability < code.Probability; + template <size_t CacheSize, typename TEntry> + struct TCache { + ui32 CacheKey[CacheSize]; + TEntry CacheVal[CacheSize]; + size_t Hits; + size_t Misses; + ui32 Hash(ui32 key) { + return key % CacheSize; + } + TCache() { + Hits = 0; + Misses = 0; + Clear(); + } + void Clear() { + for (size_t i = 0; i < CacheSize; ++i) { + ui32 j = 0; + for (; Hash(j) == i; ++j) + ; + CacheKey[i] = j; + } + } + }; + + struct TCode { + i64 Probability; + ui32 Start; + ui32 Bits; + ui32 Prefix; + ui32 PrefLength; + TCode(i64 probability = 0, ui32 start = 0, ui32 bits = 0) + : Probability(probability) + , Start(start) + , Bits(bits) + { } - bool operator>(const TCode& code) const { - return Probability > code.Probability; - } - }; - - struct TAccum { - struct TTable { - TAutoPtr<TTable> Tables[16]; - i64 Counts[16]; - TTable(const TTable& other) { - for (size_t i = 0; i < 16; ++i) { - Counts[i] = other.Counts[i]; - if (other.Tables[i].Get()) { - Tables[i].Reset(new TTable(*other.Tables[i].Get())); - } + bool operator<(const TCode& code) const { + return Probability < code.Probability; + } + + bool operator>(const TCode& code) const { + return Probability > code.Probability; + } + }; + + struct TAccum { + struct TTable { + TAutoPtr<TTable> Tables[16]; + i64 Counts[16]; + TTable(const TTable& other) { + for (size_t i = 0; i < 16; ++i) { + Counts[i] = other.Counts[i]; + if (other.Tables[i].Get()) { + Tables[i].Reset(new TTable(*other.Tables[i].Get())); + } + } + } + TTable() { + for (auto& count : Counts) + count = 0; + } + + i64 GetCellCount(size_t i) { + i64 count = Counts[i]; + if (Tables[i].Get()) { + for (size_t j = 0; j < 16; ++j) { + count += Tables[i]->GetCellCount(j); + } } - } - TTable() { - for (auto& count : Counts) - count = 0; + return count; } - i64 GetCellCount(size_t i) { - i64 count = Counts[i]; - if (Tables[i].Get()) { - for (size_t j = 0; j < 16; ++j) { - count += Tables[i]->GetCellCount(j); - } - } - return count; - } - - i64 GetCount() { - i64 count = 0; + i64 GetCount() { + i64 count = 0; for (size_t j = 0; j < 16; ++j) { - count += GetCellCount(j); + count += GetCellCount(j); } - return count; + return count; } - void GenerateFreqs(TVector<std::pair<i64, TCode>>& codes, int depth, int termDepth, ui32 code, i64 cnt) { - if (depth == termDepth) { - for (size_t i = 0; i < 16; ++i) { - i64 iCount = GetCellCount(i); - if (Tables[i].Get()) { - Counts[i] = iCount; - Tables[i].Reset(nullptr); - } - - if (iCount > cnt || (termDepth == 0 && iCount > 0)) { - std::pair<i64, TCode> codep; - codep.first = iCount; - codep.second.Probability = iCount; - codep.second.Start = code + (i << (28 - depth)); - codep.second.Bits = 28 - depth; - codes.push_back(codep); - Counts[i] = 0; - } - } - } + void GenerateFreqs(TVector<std::pair<i64, TCode>>& codes, int depth, int termDepth, ui32 code, i64 cnt) { + if (depth == termDepth) { + for (size_t i = 0; i < 16; ++i) { + i64 iCount = GetCellCount(i); + if (Tables[i].Get()) { + Counts[i] = iCount; + Tables[i].Reset(nullptr); + } + + if (iCount > cnt || (termDepth == 0 && iCount > 0)) { + std::pair<i64, TCode> codep; + codep.first = iCount; + codep.second.Probability = iCount; + codep.second.Start = code + (i << (28 - depth)); + codep.second.Bits = 28 - depth; + codes.push_back(codep); + Counts[i] = 0; + } + } + } for (size_t i = 0; i < 16; ++i) { if (Tables[i].Get()) { - Tables[i]->GenerateFreqs(codes, depth + 4, termDepth, code + (i << (28 - depth)), cnt); + Tables[i]->GenerateFreqs(codes, depth + 4, termDepth, code + (i << (28 - depth)), cnt); } } } - }; - - TTable Root; - int TableCount; - i64 Total; - ui64 Max; - - TAccum() { - TableCount = 0; - Total = 0; - Max = 0; - } - - void GenerateFreqs(TVector<std::pair<i64, TCode>>& codes, int mul) const { - TTable root(Root); - - for (int i = 28; i > 0; i -= 4) { - root.GenerateFreqs(codes, 0, i, 0, Total / mul); - } - - i64 iCount = root.GetCount(); - if (iCount == 0) - return; - std::pair<i64, TCode> codep; - codep.first = iCount; - codep.second.Probability = iCount; - codep.second.Start = 0; - ui32 bits = 0; - while (1) { - if ((1ULL << bits) > Max) - break; - ++bits; - } - codep.second.Bits = bits; - codes.push_back(codep); + }; + + TTable Root; + int TableCount; + i64 Total; + ui64 Max; + + TAccum() { + TableCount = 0; + Total = 0; + Max = 0; + } + + void GenerateFreqs(TVector<std::pair<i64, TCode>>& codes, int mul) const { + TTable root(Root); + + for (int i = 28; i > 0; i -= 4) { + root.GenerateFreqs(codes, 0, i, 0, Total / mul); + } + + i64 iCount = root.GetCount(); + if (iCount == 0) + return; + std::pair<i64, TCode> codep; + codep.first = iCount; + codep.second.Probability = iCount; + codep.second.Start = 0; + ui32 bits = 0; + while (1) { + if ((1ULL << bits) > Max) + break; + ++bits; + } + codep.second.Bits = bits; + codes.push_back(codep); } - TCache<256, i64*> Cache; - - void AddMap(ui32 value, i64 weight = 1) { - ui32 index = Cache.Hash(value); - if (Cache.CacheKey[index] == value) { - Cache.CacheVal[index][0] += weight; - return; - } - TTable* root = &Root; - for (size_t i = 0; i < 15; ++i) { - ui32 index2 = (value >> (28 - i * 4)) & 0xf; - if (!root->Tables[index2].Get()) { - if (TableCount < 1024) { - ++TableCount; - root->Tables[index2].Reset(new TTable); - } else { - Cache.CacheKey[index2] = value; - Cache.CacheVal[index2] = &root->Counts[index2]; - root->Counts[index2] += weight; - return; - } + TCache<256, i64*> Cache; + + void AddMap(ui32 value, i64 weight = 1) { + ui32 index = Cache.Hash(value); + if (Cache.CacheKey[index] == value) { + Cache.CacheVal[index][0] += weight; + return; + } + TTable* root = &Root; + for (size_t i = 0; i < 15; ++i) { + ui32 index2 = (value >> (28 - i * 4)) & 0xf; + if (!root->Tables[index2].Get()) { + if (TableCount < 1024) { + ++TableCount; + root->Tables[index2].Reset(new TTable); + } else { + Cache.CacheKey[index2] = value; + Cache.CacheVal[index2] = &root->Counts[index2]; + root->Counts[index2] += weight; + return; + } } - root = root->Tables[index2].Get(); + root = root->Tables[index2].Get(); } - - Cache.CacheKey[index] = value; - Cache.CacheVal[index] = &root->Counts[value & 0xf]; - root->Counts[value & 0xf] += weight; + + Cache.CacheKey[index] = value; + Cache.CacheVal[index] = &root->Counts[value & 0xf]; + root->Counts[value & 0xf] += weight; } - void Add(ui32 value, i64 weight = 1) { - Max = ::Max(Max, (ui64)value); - Total += weight; - AddMap(value, weight); - }; + void Add(ui32 value, i64 weight = 1) { + Max = ::Max(Max, (ui64)value); + Total += weight; + AddMap(value, weight); + }; }; - struct THuffNode { - i64 Weight; - i64 Priority; - THuffNode* Nodes[2]; - TCode* Code; - THuffNode(i64 weight, i64 priority, TCode* code) - : Weight(weight) - , Priority(priority) - , Code(code) - { - Nodes[0] = nullptr; - Nodes[1] = nullptr; - } - - void BuildPrefixes(ui32 depth, ui32 prefix) { - if (Code) { - Code->Prefix = prefix; - Code->PrefLength = depth; - return; - } - Nodes[0]->BuildPrefixes(depth + 1, prefix + (0UL << depth)); - Nodes[1]->BuildPrefixes(depth + 1, prefix + (1UL << depth)); - } - - i64 Iterate(size_t depth) const { - if (Code) { - return (depth + Code->Bits) * Code->Probability; - } - return Nodes[0]->Iterate(depth + 1) + Nodes[1]->Iterate(depth + 1); + struct THuffNode { + i64 Weight; + i64 Priority; + THuffNode* Nodes[2]; + TCode* Code; + THuffNode(i64 weight, i64 priority, TCode* code) + : Weight(weight) + , Priority(priority) + , Code(code) + { + Nodes[0] = nullptr; + Nodes[1] = nullptr; + } + + void BuildPrefixes(ui32 depth, ui32 prefix) { + if (Code) { + Code->Prefix = prefix; + Code->PrefLength = depth; + return; + } + Nodes[0]->BuildPrefixes(depth + 1, prefix + (0UL << depth)); + Nodes[1]->BuildPrefixes(depth + 1, prefix + (1UL << depth)); + } + + i64 Iterate(size_t depth) const { + if (Code) { + return (depth + Code->Bits) * Code->Probability; + } + return Nodes[0]->Iterate(depth + 1) + Nodes[1]->Iterate(depth + 1); } - size_t Depth() const { - if (Code) { - return 0; - } - return Max(Nodes[0]->Depth(), Nodes[1]->Depth()) + 1; + size_t Depth() const { + if (Code) { + return 0; + } + return Max(Nodes[0]->Depth(), Nodes[1]->Depth()) + 1; } - }; - - struct THLess { - bool operator()(const THuffNode* a, const THuffNode* b) { - if (a->Weight > b->Weight) - return 1; - if (a->Weight == b->Weight && a->Priority > b->Priority) - return 1; + }; + + struct THLess { + bool operator()(const THuffNode* a, const THuffNode* b) { + if (a->Weight > b->Weight) + return 1; + if (a->Weight == b->Weight && a->Priority > b->Priority) + return 1; return 0; } - }; - - inline i64 BuildHuff(TVector<TCode>& codes) { - TVector<TSimpleSharedPtr<THuffNode>> hold; - std::priority_queue<THuffNode*, TVector<THuffNode*>, THLess> nodes; - i64 ret = 0; - - int priority = 0; - for (size_t i = 0; i < codes.size(); ++i) { - TSimpleSharedPtr<THuffNode> node(new THuffNode(codes[i].Probability, priority++, &codes[i])); - hold.push_back(node); - nodes.push(node.Get()); - } - - while (nodes.size() > 1) { - THuffNode* nodea = nodes.top(); - nodes.pop(); - THuffNode* nodeb = nodes.top(); - nodes.pop(); - TSimpleSharedPtr<THuffNode> node(new THuffNode(nodea->Weight + nodeb->Weight, priority++, nullptr)); - node->Nodes[0] = nodea; - node->Nodes[1] = nodeb; - hold.push_back(node); - nodes.push(node.Get()); - } - - if (nodes.size()) { - THuffNode* node = nodes.top(); - node->BuildPrefixes(0, 0); - ret = node->Iterate(0); - } - - return ret; - }; - - struct TCoderEntry { - ui32 MinValue; - ui16 Prefix; - ui8 PrefixBits; - ui8 AllBits; - - ui64 MaxValue() const { - return MinValue + (1ULL << (AllBits - PrefixBits)); - } - }; - - inline i64 Analyze(const TAccum& acc, TVector<TCoderEntry>& retCodes) { - i64 ret; - for (int k = 256; k > 0; --k) { - retCodes.clear(); - TVector<std::pair<i64, TCode>> pairs; - acc.GenerateFreqs(pairs, k); - TVector<TCode> codes; - for (size_t i = 0; i < pairs.size(); ++i) { - codes.push_back(pairs[i].second); - } - - StableSort(codes.begin(), codes.end(), std::greater<TCode>()); - - ret = BuildHuff(codes); - bool valid = true; - for (size_t i = 0; i < codes.size(); ++i) { - TCoderEntry code; - code.MinValue = codes[i].Start; - code.Prefix = codes[i].Prefix; - code.PrefixBits = codes[i].PrefLength; - if (code.PrefixBits > 6) - valid = false; - code.AllBits = code.PrefixBits + codes[i].Bits; - retCodes.push_back(code); - } - if (valid) - return ret; - } - - return ret; - } - - struct TComparer { - bool operator()(const TCoderEntry& e0, const TCoderEntry& e1) const { - return e0.AllBits < e1.AllBits; + }; + + inline i64 BuildHuff(TVector<TCode>& codes) { + TVector<TSimpleSharedPtr<THuffNode>> hold; + std::priority_queue<THuffNode*, TVector<THuffNode*>, THLess> nodes; + i64 ret = 0; + + int priority = 0; + for (size_t i = 0; i < codes.size(); ++i) { + TSimpleSharedPtr<THuffNode> node(new THuffNode(codes[i].Probability, priority++, &codes[i])); + hold.push_back(node); + nodes.push(node.Get()); + } + + while (nodes.size() > 1) { + THuffNode* nodea = nodes.top(); + nodes.pop(); + THuffNode* nodeb = nodes.top(); + nodes.pop(); + TSimpleSharedPtr<THuffNode> node(new THuffNode(nodea->Weight + nodeb->Weight, priority++, nullptr)); + node->Nodes[0] = nodea; + node->Nodes[1] = nodeb; + hold.push_back(node); + nodes.push(node.Get()); + } + + if (nodes.size()) { + THuffNode* node = nodes.top(); + node->BuildPrefixes(0, 0); + ret = node->Iterate(0); + } + + return ret; + }; + + struct TCoderEntry { + ui32 MinValue; + ui16 Prefix; + ui8 PrefixBits; + ui8 AllBits; + + ui64 MaxValue() const { + return MinValue + (1ULL << (AllBits - PrefixBits)); + } + }; + + inline i64 Analyze(const TAccum& acc, TVector<TCoderEntry>& retCodes) { + i64 ret; + for (int k = 256; k > 0; --k) { + retCodes.clear(); + TVector<std::pair<i64, TCode>> pairs; + acc.GenerateFreqs(pairs, k); + TVector<TCode> codes; + for (size_t i = 0; i < pairs.size(); ++i) { + codes.push_back(pairs[i].second); + } + + StableSort(codes.begin(), codes.end(), std::greater<TCode>()); + + ret = BuildHuff(codes); + bool valid = true; + for (size_t i = 0; i < codes.size(); ++i) { + TCoderEntry code; + code.MinValue = codes[i].Start; + code.Prefix = codes[i].Prefix; + code.PrefixBits = codes[i].PrefLength; + if (code.PrefixBits > 6) + valid = false; + code.AllBits = code.PrefixBits + codes[i].Bits; + retCodes.push_back(code); + } + if (valid) + return ret; } - }; - struct TCoder { - TVector<TCoderEntry> Entries; - void Normalize() { - TComparer comp; - StableSort(Entries.begin(), Entries.end(), comp); - } - TCoder() { - InitDefault(); - } - void InitDefault() { - ui64 cum = 0; - Cache.Clear(); - Entries.clear(); - ui16 b = 1; - for (ui16 i = 0; i < 40; ++i) { - ui16 bits = Min(b, (ui16)(32)); - b = (b * 16) / 10 + 1; - if (b > 32) - b = 32; - TCoderEntry entry; - entry.PrefixBits = i + 1; - entry.AllBits = entry.PrefixBits + bits; - entry.MinValue = (ui32)Min(cum, (ui64)(ui32)(-1)); - cum += (1ULL << bits); - entry.Prefix = ((1UL << i) - 1); - Entries.push_back(entry); - if (cum > (ui32)(-1)) { - return; - } - }; - } + return ret; + } - TCache<1024, TCoderEntry> Cache; - - ui64 RealCode(ui32 value, const TCoderEntry& entry, size_t& length) { - length = entry.AllBits; - return (ui64(value - entry.MinValue) << entry.PrefixBits) + entry.Prefix; - } - - bool Empty() const { - return Entries.empty(); + struct TComparer { + bool operator()(const TCoderEntry& e0, const TCoderEntry& e1) const { + return e0.AllBits < e1.AllBits; } - const TCoderEntry& GetEntry(ui32 code, ui8& id) const { - for (size_t i = 0; i < Entries.size(); ++i) { - const TCoderEntry& entry = Entries[i]; - ui32 prefMask = (1UL << entry.PrefixBits) - 1UL; - if (entry.Prefix == (code & prefMask)) { - id = ui8(i); - return entry; - } + }; + + struct TCoder { + TVector<TCoderEntry> Entries; + void Normalize() { + TComparer comp; + StableSort(Entries.begin(), Entries.end(), comp); + } + TCoder() { + InitDefault(); + } + void InitDefault() { + ui64 cum = 0; + Cache.Clear(); + Entries.clear(); + ui16 b = 1; + for (ui16 i = 0; i < 40; ++i) { + ui16 bits = Min(b, (ui16)(32)); + b = (b * 16) / 10 + 1; + if (b > 32) + b = 32; + TCoderEntry entry; + entry.PrefixBits = i + 1; + entry.AllBits = entry.PrefixBits + bits; + entry.MinValue = (ui32)Min(cum, (ui64)(ui32)(-1)); + cum += (1ULL << bits); + entry.Prefix = ((1UL << i) - 1); + Entries.push_back(entry); + if (cum > (ui32)(-1)) { + return; + } + }; + } + + TCache<1024, TCoderEntry> Cache; + + ui64 RealCode(ui32 value, const TCoderEntry& entry, size_t& length) { + length = entry.AllBits; + return (ui64(value - entry.MinValue) << entry.PrefixBits) + entry.Prefix; + } + + bool Empty() const { + return Entries.empty(); + } + const TCoderEntry& GetEntry(ui32 code, ui8& id) const { + for (size_t i = 0; i < Entries.size(); ++i) { + const TCoderEntry& entry = Entries[i]; + ui32 prefMask = (1UL << entry.PrefixBits) - 1UL; + if (entry.Prefix == (code & prefMask)) { + id = ui8(i); + return entry; + } } - ythrow yexception() << "bad entry"; - return Entries[0]; + ythrow yexception() << "bad entry"; + return Entries[0]; } - ui64 Code(ui32 entry, size_t& length) { - ui32 index = Cache.Hash(entry); - if (Cache.CacheKey[index] == entry) { - ++Cache.Hits; + ui64 Code(ui32 entry, size_t& length) { + ui32 index = Cache.Hash(entry); + if (Cache.CacheKey[index] == entry) { + ++Cache.Hits; return RealCode(entry, Cache.CacheVal[index], length); } - ++Cache.Misses; - for (size_t i = 0; i < Entries.size(); ++i) { - if (entry >= Entries[i].MinValue && entry < Entries[i].MaxValue()) { - Cache.CacheKey[index] = entry; - Cache.CacheVal[index] = Entries[i]; - return RealCode(entry, Cache.CacheVal[index], length); - } - } - - ythrow yexception() << "bad huff tree"; - return 0; + ++Cache.Misses; + for (size_t i = 0; i < Entries.size(); ++i) { + if (entry >= Entries[i].MinValue && entry < Entries[i].MaxValue()) { + Cache.CacheKey[index] = entry; + Cache.CacheVal[index] = Entries[i]; + return RealCode(entry, Cache.CacheVal[index], length); + } + } + + ythrow yexception() << "bad huff tree"; + return 0; } - }; + }; -} +} diff --git a/library/cpp/compproto/metainfo.h b/library/cpp/compproto/metainfo.h index 6e68f86e12..5675de76bc 100644 --- a/library/cpp/compproto/metainfo.h +++ b/library/cpp/compproto/metainfo.h @@ -10,295 +10,295 @@ #include "compressor.h" namespace NCompProto { - const size_t MAX_ELEMENTS = 32; + const size_t MAX_ELEMENTS = 32; - struct TScalarDefaultValue { - TScalarDefaultValue() - : Type(None) - , Value(0) - { - } - enum EType { - None = 0, - Fixed = 1, - }; - EType Type; - ui32 Value; + struct TScalarDefaultValue { + TScalarDefaultValue() + : Type(None) + , Value(0) + { + } + enum EType { + None = 0, + Fixed = 1, + }; + EType Type; + ui32 Value; }; - template <class X> - struct TMetaInfo { - X Index; - X Count; - X Mask; - X Scalar[MAX_ELEMENTS]; - TAtomicSharedPtr<TMetaInfo> Repeated[MAX_ELEMENTS]; - TScalarDefaultValue Default[MAX_ELEMENTS]; - - ui32 ScalarMask; - ui32 RepeatedMask; - size_t Size; - TString Name; - TString ChildName[MAX_ELEMENTS]; - size_t Id; - TMetaInfo* Parent; - - struct TCheck { - ui32 Count; - ui32 Mask; - ui32 LastIndex; - ui32 Check(size_t id, size_t rule) { - ui32 mask = 1UL << id; - if (Mask & mask) { - ythrow yexception() << "Mask & mask"; - } - Mask |= mask; - Y_ENSURE(rule & mask, "!(rule & mask)"); - return mask; - } - void ClearCount() { - Count = 0; - } - void ClearMask() { - Mask = 0; - } - void ClearIndex() { - LastIndex = ui32(-1); - } - void ClearAll() { - ClearCount(); - ClearIndex(); - ClearMask(); - } - TCheck() { - ClearAll(); - } - }; + template <class X> + struct TMetaInfo { + X Index; + X Count; + X Mask; + X Scalar[MAX_ELEMENTS]; + TAtomicSharedPtr<TMetaInfo> Repeated[MAX_ELEMENTS]; + TScalarDefaultValue Default[MAX_ELEMENTS]; - TCheck Serializer; + ui32 ScalarMask; + ui32 RepeatedMask; + size_t Size; + TString Name; + TString ChildName[MAX_ELEMENTS]; + size_t Id; + TMetaInfo* Parent; - void SetDefaults(TMetaInfo* parent) { - ScalarMask = 0; - RepeatedMask = 0; - Size = 0; - Id = 0; - Parent = parent; - } + struct TCheck { + ui32 Count; + ui32 Mask; + ui32 LastIndex; + ui32 Check(size_t id, size_t rule) { + ui32 mask = 1UL << id; + if (Mask & mask) { + ythrow yexception() << "Mask & mask"; + } + Mask |= mask; + Y_ENSURE(rule & mask, "!(rule & mask)"); + return mask; + } + void ClearCount() { + Count = 0; + } + void ClearMask() { + Mask = 0; + } + void ClearIndex() { + LastIndex = ui32(-1); + } + void ClearAll() { + ClearCount(); + ClearIndex(); + ClearMask(); + } + TCheck() { + ClearAll(); + } + }; - template <class T, class TBuilder> - TMetaInfo(const TMetaInfo<T>& meta, const TBuilder& builder) { - SetDefaults(nullptr); - Name = meta.Name; - builder.Build(Index, meta.Index); - builder.Build(Count, meta.Count); - builder.Build(Mask, meta.Mask); - Size = meta.Size; - Id = meta.Id; - ScalarMask = meta.ScalarMask; - RepeatedMask = meta.RepeatedMask; - for (size_t i = 0; i < Size; ++i) { - ui32 index = (1UL << i); - if (ScalarMask & index) { - builder.Build(Scalar[i], meta.Scalar[i]); - ChildName[i] = meta.ChildName[i]; - Default[i] = meta.Default[i]; - } - if (RepeatedMask & index) { - THolder<TMetaInfo> info(new TMetaInfo(*meta.Repeated[i], builder)); - info->Parent = this; - Repeated[i].Reset(info.Release()); - } + TCheck Serializer; + + void SetDefaults(TMetaInfo* parent) { + ScalarMask = 0; + RepeatedMask = 0; + Size = 0; + Id = 0; + Parent = parent; + } + + template <class T, class TBuilder> + TMetaInfo(const TMetaInfo<T>& meta, const TBuilder& builder) { + SetDefaults(nullptr); + Name = meta.Name; + builder.Build(Index, meta.Index); + builder.Build(Count, meta.Count); + builder.Build(Mask, meta.Mask); + Size = meta.Size; + Id = meta.Id; + ScalarMask = meta.ScalarMask; + RepeatedMask = meta.RepeatedMask; + for (size_t i = 0; i < Size; ++i) { + ui32 index = (1UL << i); + if (ScalarMask & index) { + builder.Build(Scalar[i], meta.Scalar[i]); + ChildName[i] = meta.ChildName[i]; + Default[i] = meta.Default[i]; + } + if (RepeatedMask & index) { + THolder<TMetaInfo> info(new TMetaInfo(*meta.Repeated[i], builder)); + info->Parent = this; + Repeated[i].Reset(info.Release()); + } } } - - TMetaInfo(TMetaInfo* parent) { - SetDefaults(parent); + + TMetaInfo(TMetaInfo* parent) { + SetDefaults(parent); } - - template <class T> - TMetaInfo& BeginRepeated(size_t id, T& functor) { - Serializer.Check(id, RepeatedMask); - TMetaInfo& res = *Repeated[id].Get(); - res.Count.BeginElement(functor); - return res; + + template <class T> + TMetaInfo& BeginRepeated(size_t id, T& functor) { + Serializer.Check(id, RepeatedMask); + TMetaInfo& res = *Repeated[id].Get(); + res.Count.BeginElement(functor); + return res; } - - template <class T> - void BeginSelf(T& functor) { - Serializer.ClearAll(); - Count.BeginElement(functor); + + template <class T> + void BeginSelf(T& functor) { + Serializer.ClearAll(); + Count.BeginElement(functor); } - - template <class T> - void EndRepeated(T& functor) { - Count.AddDelayed(Serializer.Count, functor); - Serializer.ClearCount(); - Serializer.ClearIndex(); - Y_ENSURE(Serializer.Mask == 0, "Serializer.Mask != 0"); + + template <class T> + void EndRepeated(T& functor) { + Count.AddDelayed(Serializer.Count, functor); + Serializer.ClearCount(); + Serializer.ClearIndex(); + Y_ENSURE(Serializer.Mask == 0, "Serializer.Mask != 0"); } - - template <class T> - void BeginElement(ui32 index, T& functor) { - Y_ENSURE(!(index < Serializer.LastIndex + 1), - "Name: " << Name.Quote() << " error: index < Serializer.LastIndex + 1; " - << "index: " << index << " LastIndex: " << Serializer.LastIndex); - Index.Add(index - Serializer.LastIndex, functor); - Mask.BeginElement(functor); - Serializer.LastIndex = index; - ++Serializer.Count; + + template <class T> + void BeginElement(ui32 index, T& functor) { + Y_ENSURE(!(index < Serializer.LastIndex + 1), + "Name: " << Name.Quote() << " error: index < Serializer.LastIndex + 1; " + << "index: " << index << " LastIndex: " << Serializer.LastIndex); + Index.Add(index - Serializer.LastIndex, functor); + Mask.BeginElement(functor); + Serializer.LastIndex = index; + ++Serializer.Count; } - template <class TFunctor> - void Iterate(TFunctor& functor) { - Cout << Name << " " - << "Index" << Endl; - functor.Process(Index); - Cout << "Count" << Endl; - functor.Process(Count); - Cout << "Mask" << Endl; - functor.Process(Mask); + template <class TFunctor> + void Iterate(TFunctor& functor) { + Cout << Name << " " + << "Index" << Endl; + functor.Process(Index); + Cout << "Count" << Endl; + functor.Process(Count); + Cout << "Mask" << Endl; + functor.Process(Mask); - for (size_t i = 0; i < Size; ++i) { - ui32 index = (1UL << i); - if (ScalarMask & index) { - Cout << "scalar" - << " " << i << Endl; - functor.Process(Scalar[i]); - } - if (RepeatedMask & index) { - Cout << "repeated" - << " " << i << Endl; - Repeated[i]->Iterate(functor); - } + for (size_t i = 0; i < Size; ++i) { + ui32 index = (1UL << i); + if (ScalarMask & index) { + Cout << "scalar" + << " " << i << Endl; + functor.Process(Scalar[i]); + } + if (RepeatedMask & index) { + Cout << "repeated" + << " " << i << Endl; + Repeated[i]->Iterate(functor); + } } } - template <class T> - void EndElement(T& functor) { - Mask.AddDelayed(Serializer.Mask, functor); - Serializer.ClearMask(); - } + template <class T> + void EndElement(T& functor) { + Mask.AddDelayed(Serializer.Mask, functor); + Serializer.ClearMask(); + } - template <class T> - void SetScalar(size_t id, ui32 value, T& functor) { - if (Default[id].Type != TScalarDefaultValue::Fixed || value != Default[id].Value) { - Serializer.Check(id, ScalarMask); - Scalar[id].Add(value, functor); - } - } + template <class T> + void SetScalar(size_t id, ui32 value, T& functor) { + if (Default[id].Type != TScalarDefaultValue::Fixed || value != Default[id].Value) { + Serializer.Check(id, ScalarMask); + Scalar[id].Add(value, functor); + } + } - ui32 Check(size_t id) { - Y_ENSURE(id < MAX_ELEMENTS, "id >= MAX_ELEMENTS"); + ui32 Check(size_t id) { + Y_ENSURE(id < MAX_ELEMENTS, "id >= MAX_ELEMENTS"); - ui32 mask = 1UL << id; - if (ScalarMask & mask) { - ythrow yexception() << "ScalarMask & mask"; + ui32 mask = 1UL << id; + if (ScalarMask & mask) { + ythrow yexception() << "ScalarMask & mask"; } - if (RepeatedMask & mask) { - ythrow yexception() << "RepeatedMask & mask"; + if (RepeatedMask & mask) { + ythrow yexception() << "RepeatedMask & mask"; } - Size = Max(id + 1, Size); - return mask; + Size = Max(id + 1, Size); + return mask; } - TMetaInfo(IInputStream& stream) { - SetDefaults(nullptr); - Load(stream); + TMetaInfo(IInputStream& stream) { + SetDefaults(nullptr); + Load(stream); } - TMetaInfo(const TString& str) { - SetDefaults(nullptr); - TStringInput stream(str); - Load(stream); + TMetaInfo(const TString& str) { + SetDefaults(nullptr); + TStringInput stream(str); + Load(stream); } - void Save(IOutputStream& stream, const TString& offset = TString()) { - stream << offset << "repeated " << Name << " id " << Id << Endl; - TString step = " "; - stream << step << offset << "index" << Endl; - Index.Save(stream, step + offset); - stream << step << offset << "count" << Endl; - Count.Save(stream, step + offset); - stream << step << offset << "mask" << Endl; - Mask.Save(stream, step + offset); + void Save(IOutputStream& stream, const TString& offset = TString()) { + stream << offset << "repeated " << Name << " id " << Id << Endl; + TString step = " "; + stream << step << offset << "index" << Endl; + Index.Save(stream, step + offset); + stream << step << offset << "count" << Endl; + Count.Save(stream, step + offset); + stream << step << offset << "mask" << Endl; + Mask.Save(stream, step + offset); - for (size_t i = 0; i < MAX_ELEMENTS; ++i) { - ui32 mask = 1UL << i; - if (mask & RepeatedMask) { - Repeated[i]->Save(stream, step + offset); - } else if (mask & ScalarMask) { - stream << step << offset << "scalar " << ChildName[i] << " id " << i; - stream << " default " << (Default[i].Type == TScalarDefaultValue::None ? " no " : " const "); - if (Default[i].Type == TScalarDefaultValue::Fixed) { - stream << Default[i].Value; - } - stream << Endl; - stream << step << offset << "table " - << "id " << i << Endl; - Scalar[i].Save(stream, step + offset); + for (size_t i = 0; i < MAX_ELEMENTS; ++i) { + ui32 mask = 1UL << i; + if (mask & RepeatedMask) { + Repeated[i]->Save(stream, step + offset); + } else if (mask & ScalarMask) { + stream << step << offset << "scalar " << ChildName[i] << " id " << i; + stream << " default " << (Default[i].Type == TScalarDefaultValue::None ? " no " : " const "); + if (Default[i].Type == TScalarDefaultValue::Fixed) { + stream << Default[i].Value; + } + stream << Endl; + stream << step << offset << "table " + << "id " << i << Endl; + Scalar[i].Save(stream, step + offset); } } - stream << offset << "end" << Endl; + stream << offset << "end" << Endl; } - void Load(IInputStream& stream) { - TString name; - stream >> name; - if (name == "repeated") { - stream >> name; - } - Name = name; + void Load(IInputStream& stream) { + TString name; stream >> name; - Y_ENSURE(name == "id", "Name mismatch: " << name.Quote() << " != id. "); - stream >> Id; + if (name == "repeated") { + stream >> name; + } + Name = name; + stream >> name; + Y_ENSURE(name == "id", "Name mismatch: " << name.Quote() << " != id. "); + stream >> Id; - while (1) { + while (1) { stream >> name; - if (name == "index") { - Index.Load(stream); - } else if (name == "count") { - Count.Load(stream); - } else if (name == "mask") { - Mask.Load(stream); - } else if (name == "table") { - stream >> name; - Y_ENSURE(name == "id", "Name mismatch: " << name.Quote() << " != id. "); - size_t id; - stream >> id; - Scalar[id].Load(stream); - } else if (name == "repeated") { - THolder<TMetaInfo> info(new TMetaInfo(this)); - info->Load(stream); - size_t id = info->Id; - RepeatedMask |= Check(id); - Repeated[id].Reset(info.Release()); - } else if (name == "scalar") { - stream >> name; - TString childName = name; - stream >> name; - Y_ENSURE(name == "id", "Name mismatch: " << name.Quote() << " != id. "); - size_t id; - stream >> id; - ScalarMask |= Check(id); - ChildName[id] = childName; - stream >> name; - Y_ENSURE(name == "default", "Name mismatch: " << name.Quote() << " != default. "); - stream >> name; - if (name == "no") { - Default[id].Type = TScalarDefaultValue::None; - } else if (name == "const") { - ui32 def; - stream >> def; - Default[id].Type = TScalarDefaultValue::Fixed; - Default[id].Value = def; - } else { - ythrow yexception() << "Unsupported default value specification: " << name.Quote(); - } - } else if (name == "end" /* || stream.IsEOF()*/) { - return; + if (name == "index") { + Index.Load(stream); + } else if (name == "count") { + Count.Load(stream); + } else if (name == "mask") { + Mask.Load(stream); + } else if (name == "table") { + stream >> name; + Y_ENSURE(name == "id", "Name mismatch: " << name.Quote() << " != id. "); + size_t id; + stream >> id; + Scalar[id].Load(stream); + } else if (name == "repeated") { + THolder<TMetaInfo> info(new TMetaInfo(this)); + info->Load(stream); + size_t id = info->Id; + RepeatedMask |= Check(id); + Repeated[id].Reset(info.Release()); + } else if (name == "scalar") { + stream >> name; + TString childName = name; + stream >> name; + Y_ENSURE(name == "id", "Name mismatch: " << name.Quote() << " != id. "); + size_t id; + stream >> id; + ScalarMask |= Check(id); + ChildName[id] = childName; + stream >> name; + Y_ENSURE(name == "default", "Name mismatch: " << name.Quote() << " != default. "); + stream >> name; + if (name == "no") { + Default[id].Type = TScalarDefaultValue::None; + } else if (name == "const") { + ui32 def; + stream >> def; + Default[id].Type = TScalarDefaultValue::Fixed; + Default[id].Value = def; + } else { + ythrow yexception() << "Unsupported default value specification: " << name.Quote(); + } + } else if (name == "end" /* || stream.IsEOF()*/) { + return; } } } - }; + }; -} +} diff --git a/library/cpp/comptable/comptable.cpp b/library/cpp/comptable/comptable.cpp index 8a92d4d1aa..8e87da3060 100644 --- a/library/cpp/comptable/comptable.cpp +++ b/library/cpp/comptable/comptable.cpp @@ -11,433 +11,433 @@ #include <cstdlib> namespace NCompTable { - static const ui32 magicHashMul = 0xd077cd1f; - static const size_t hashSizeLog = 18; - static const size_t hashSize = 1 << hashSizeLog; + static const ui32 magicHashMul = 0xd077cd1f; + static const size_t hashSizeLog = 18; + static const size_t hashSize = 1 << hashSizeLog; - size_t HashIndex(ui32 value, ui32 hashMul) { - return (value * hashMul) >> (32 - hashSizeLog); - } + size_t HashIndex(ui32 value, ui32 hashMul) { + return (value * hashMul) >> (32 - hashSizeLog); + } - void TCompressorTable::GetHuffCode(const NCompProto::TCoderEntry& entry, ui32 value, ui64& bitCode, ui8& bitLength) const { - ui64 code = value - entry.MinValue; - bitCode = (code << entry.PrefixBits) | entry.Prefix; - bitLength = entry.AllBits; - } - void TCompressorTable::GetLastHuffCode(ui32 value, ui64& bitCode, ui8& bitLength) const { - GetHuffCode(HuffCodes[9], value, bitCode, bitLength); - } - void TCompressorTable::GetHuffCode(ui32 value, ui64& bitCode, ui8& bitLength) const { - for (auto huffCode : HuffCodes) { - ui32 minValue = huffCode.MinValue; - if (minValue <= value && value < huffCode.MaxValue()) { - GetHuffCode(huffCode, value, bitCode, bitLength); - return; - } + void TCompressorTable::GetHuffCode(const NCompProto::TCoderEntry& entry, ui32 value, ui64& bitCode, ui8& bitLength) const { + ui64 code = value - entry.MinValue; + bitCode = (code << entry.PrefixBits) | entry.Prefix; + bitLength = entry.AllBits; + } + void TCompressorTable::GetLastHuffCode(ui32 value, ui64& bitCode, ui8& bitLength) const { + GetHuffCode(HuffCodes[9], value, bitCode, bitLength); + } + void TCompressorTable::GetHuffCode(ui32 value, ui64& bitCode, ui8& bitLength) const { + for (auto huffCode : HuffCodes) { + ui32 minValue = huffCode.MinValue; + if (minValue <= value && value < huffCode.MaxValue()) { + GetHuffCode(huffCode, value, bitCode, bitLength); + return; + } } - abort(); + abort(); } - ui8 TCompressorTable::GetHuffIndex(ui8 prefix) { - for (size_t i = 0; i < 10; ++i) { - if ((prefix & ((1 << HuffCodes[i].PrefixBits) - 1)) == HuffCodes[i].Prefix) { - return i; - } + ui8 TCompressorTable::GetHuffIndex(ui8 prefix) { + for (size_t i = 0; i < 10; ++i) { + if ((prefix & ((1 << HuffCodes[i].PrefixBits) - 1)) == HuffCodes[i].Prefix) { + return i; + } } - abort(); - return 0; + abort(); + return 0; } - void TCompressorTable::BuildHuffCodes(i64 totalFreq, i64 freqs[65536]) { - i64 add = 1; - while (1) { - if (BuildHuffCodes(totalFreq, freqs, add)) { - return; - } - add = add * 2; + void TCompressorTable::BuildHuffCodes(i64 totalFreq, i64 freqs[65536]) { + i64 add = 1; + while (1) { + if (BuildHuffCodes(totalFreq, freqs, add)) { + return; + } + add = add * 2; } } - bool TCompressorTable::BuildHuffCodes(i64 totalFreq, i64 freqs[65536], i64 add) { - TVector<NCompProto::TCode> codes; - size_t bits[] = {0, 1, 2, 4, 8, 10, 12, 14, 16, 32}; - size_t off = 0; - ui32 total = totalFreq; - for (size_t i = 0; i < 9; ++i) { - size_t size = 1 << bits[i]; - ui32 weight = 0; - for (size_t j = off; j < off + size && j < 65536; ++j) { - weight += freqs[j]; - } - codes.push_back(NCompProto::TCode(weight + add, ui32(off), bits[i])); - total = total > weight ? total - weight : 0; - off += size; + bool TCompressorTable::BuildHuffCodes(i64 totalFreq, i64 freqs[65536], i64 add) { + TVector<NCompProto::TCode> codes; + size_t bits[] = {0, 1, 2, 4, 8, 10, 12, 14, 16, 32}; + size_t off = 0; + ui32 total = totalFreq; + for (size_t i = 0; i < 9; ++i) { + size_t size = 1 << bits[i]; + ui32 weight = 0; + for (size_t j = off; j < off + size && j < 65536; ++j) { + weight += freqs[j]; + } + codes.push_back(NCompProto::TCode(weight + add, ui32(off), bits[i])); + total = total > weight ? total - weight : 0; + off += size; } - codes.push_back(NCompProto::TCode(total + add, 0, 32)); - i64 ret = NCompProto::BuildHuff(codes); - Y_UNUSED(ret); - for (size_t i = 0; i < codes.size(); ++i) { - NCompProto::TCoderEntry& code = HuffCodes[i]; - code.MinValue = codes[i].Start; - code.Prefix = codes[i].Prefix; - code.PrefixBits = codes[i].PrefLength; - code.AllBits = code.PrefixBits + codes[i].Bits; - if (code.PrefixBits > 8) { - return false; - } - } - for (size_t i = 0; i < 256; ++i) { - HuffIndices[i] = GetHuffIndex(ui8(i)); + codes.push_back(NCompProto::TCode(total + add, 0, 32)); + i64 ret = NCompProto::BuildHuff(codes); + Y_UNUSED(ret); + for (size_t i = 0; i < codes.size(); ++i) { + NCompProto::TCoderEntry& code = HuffCodes[i]; + code.MinValue = codes[i].Start; + code.Prefix = codes[i].Prefix; + code.PrefixBits = codes[i].PrefLength; + code.AllBits = code.PrefixBits + codes[i].Bits; + if (code.PrefixBits > 8) { + return false; + } } - return true; + for (size_t i = 0; i < 256; ++i) { + HuffIndices[i] = GetHuffIndex(ui8(i)); + } + return true; } - template <class TIterator> - void Iterate(const TStringBuf& data, TIterator& iterator) { - size_t i = 0; - iterator.Visit(ui32(data.size())); - for (; i + 3 < data.size(); i += 4) { + template <class TIterator> + void Iterate(const TStringBuf& data, TIterator& iterator) { + size_t i = 0; + iterator.Visit(ui32(data.size())); + for (; i + 3 < data.size(); i += 4) { iterator.Visit(reinterpret_cast<const ui32*>(data.data() + i)[0]); - } - if (i != data.size()) { - ui32 buffer[1] = {0}; + } + if (i != data.size()) { + ui32 buffer[1] = {0}; memcpy(buffer, data.data() + i, data.size() - i); - iterator.Visit(buffer[0]); - } + iterator.Visit(buffer[0]); + } } - class TDataCompressor { - ui32 Keys[hashSize]; - ui32 Vals[hashSize]; - ui64 BitCodes[hashSize]; - ui8 BitLengths[hashSize]; - ui32 HashMul; - const TCompressorTable Table; - - public: - TDataCompressor(const TCompressorTable& table) - : Table(table) - { - HashMul = table.HashMul; - for (size_t i = 0; i < hashSize; ++i) { - Keys[i] = 0; - Vals[i] = ui32(-1); - } - for (size_t i = 0; i < 65536; ++i) { - size_t index = HashIndex(table.Table[i], table.HashMul); - Keys[index] = table.Table[i]; - Vals[index] = i; - table.GetHuffCode(ui32(i), BitCodes[index], BitLengths[index]); - } + class TDataCompressor { + ui32 Keys[hashSize]; + ui32 Vals[hashSize]; + ui64 BitCodes[hashSize]; + ui8 BitLengths[hashSize]; + ui32 HashMul; + const TCompressorTable Table; + + public: + TDataCompressor(const TCompressorTable& table) + : Table(table) + { + HashMul = table.HashMul; + for (size_t i = 0; i < hashSize; ++i) { + Keys[i] = 0; + Vals[i] = ui32(-1); + } + for (size_t i = 0; i < 65536; ++i) { + size_t index = HashIndex(table.Table[i], table.HashMul); + Keys[index] = table.Table[i]; + Vals[index] = i; + table.GetHuffCode(ui32(i), BitCodes[index], BitLengths[index]); + } } - void GetStat(ui32 val, ui32& stat) const { - size_t index = HashIndex(val, HashMul); - if (Keys[index] == val) { - stat = Vals[index]; - } else { - stat = ui32(-1); - } + void GetStat(ui32 val, ui32& stat) const { + size_t index = HashIndex(val, HashMul); + if (Keys[index] == val) { + stat = Vals[index]; + } else { + stat = ui32(-1); + } } - void GetStat(ui32 val, ui64& bitCode, ui8& bitLength) const { - size_t index = HashIndex(val, HashMul); - if (Keys[index] == val) { - bitCode = BitCodes[index]; - bitLength = BitLengths[index]; - } else { - Table.GetLastHuffCode(val, bitCode, bitLength); - } + void GetStat(ui32 val, ui64& bitCode, ui8& bitLength) const { + size_t index = HashIndex(val, HashMul); + if (Keys[index] == val) { + bitCode = BitCodes[index]; + bitLength = BitLengths[index]; + } else { + Table.GetLastHuffCode(val, bitCode, bitLength); + } } - size_t Compress4(const ui32* data4, ui8* dataBuff) const { - ui8* oldBuff = dataBuff; - ++dataBuff; - ui32 status = 0; - for (size_t i = 0; i < 4; ++i) { - status = (status << 2); - ui32 data = data4[i]; - if (data == 0) { - continue; - } - ui32 stat; - GetStat(data, stat); - if (stat < 256) { - memcpy(dataBuff, &stat, 1); - dataBuff += 1; - status += 1; - } else if (stat < 65536) { - memcpy(dataBuff, &stat, 2); - dataBuff += 2; - status += 2; - } else { - memcpy(dataBuff, &data, 4); - dataBuff += 4; - status += 3; - } - } - oldBuff[0] = ui8(status); - return dataBuff - oldBuff; + size_t Compress4(const ui32* data4, ui8* dataBuff) const { + ui8* oldBuff = dataBuff; + ++dataBuff; + ui32 status = 0; + for (size_t i = 0; i < 4; ++i) { + status = (status << 2); + ui32 data = data4[i]; + if (data == 0) { + continue; + } + ui32 stat; + GetStat(data, stat); + if (stat < 256) { + memcpy(dataBuff, &stat, 1); + dataBuff += 1; + status += 1; + } else if (stat < 65536) { + memcpy(dataBuff, &stat, 2); + dataBuff += 2; + status += 2; + } else { + memcpy(dataBuff, &data, 4); + dataBuff += 4; + status += 3; + } + } + oldBuff[0] = ui8(status); + return dataBuff - oldBuff; } - struct TCompressorIterator { - TVector<char>& Result; - const TDataCompressor& Compressor; - ui32 Cached[4]; - size_t Index; - size_t RealSize; - TCompressorIterator(TVector<char>& result, const TDataCompressor& compressor) - : Result(result) - , Compressor(compressor) - , Index(0) - , RealSize(0) - { - } - void Flush() { - Result.yresize(RealSize + 32); - RealSize += Compressor.Compress4(Cached, reinterpret_cast<ui8*>(Result.data()) + RealSize); - Index = 0; - } - void Visit(const ui32 data) { - Cached[Index] = data; - ++Index; - if (Index == 4) { - Flush(); - } - } - ~TCompressorIterator() { - if (Index != 0) { - for (size_t i = Index; i < 4; ++i) { - Cached[i] = 0; - } - Flush(); - } - Result.yresize(RealSize); + struct TCompressorIterator { + TVector<char>& Result; + const TDataCompressor& Compressor; + ui32 Cached[4]; + size_t Index; + size_t RealSize; + TCompressorIterator(TVector<char>& result, const TDataCompressor& compressor) + : Result(result) + , Compressor(compressor) + , Index(0) + , RealSize(0) + { } - }; - struct THQCompressorIterator { - TVector<char>& Result; - const TDataCompressor& Compressor; - size_t RealSize; - ui64 Offset; - THQCompressorIterator(TVector<char>& result, const TDataCompressor& compressor) - : Result(result) - , Compressor(compressor) - , RealSize(0) - , Offset(0) - { - } - void Visit(const ui32 data) { - size_t byteOff = Offset >> 3; - Result.yresize(byteOff + 32); - ui64 bitCode; - ui8 bitLength; - Compressor.GetStat(data, bitCode, bitLength); - ui64 dst; - memcpy(&dst, &Result[byteOff], sizeof(dst)); - ui64 mask = ((1ULL << (Offset & 7)) - 1ULL); - dst = (dst & mask) | (bitCode << (Offset & 7)); - memcpy(&Result[byteOff], &dst, sizeof(dst)); - Offset += bitLength; - } - ~THQCompressorIterator() { - Result.yresize((Offset + 7) >> 3); - if (Offset & 7) - Result.back() &= (1 << (Offset & 7)) - 1; - } - }; - template <bool HQ> - void Compress(const TStringBuf& stringBuf, TVector<char>& rslt) const { - if (!HQ) { - TCompressorIterator iterator(rslt, *this); - Iterate(stringBuf, iterator); + void Flush() { + Result.yresize(RealSize + 32); + RealSize += Compressor.Compress4(Cached, reinterpret_cast<ui8*>(Result.data()) + RealSize); + Index = 0; + } + void Visit(const ui32 data) { + Cached[Index] = data; + ++Index; + if (Index == 4) { + Flush(); + } + } + ~TCompressorIterator() { + if (Index != 0) { + for (size_t i = Index; i < 4; ++i) { + Cached[i] = 0; + } + Flush(); + } + Result.yresize(RealSize); + } + }; + struct THQCompressorIterator { + TVector<char>& Result; + const TDataCompressor& Compressor; + size_t RealSize; + ui64 Offset; + THQCompressorIterator(TVector<char>& result, const TDataCompressor& compressor) + : Result(result) + , Compressor(compressor) + , RealSize(0) + , Offset(0) + { + } + void Visit(const ui32 data) { + size_t byteOff = Offset >> 3; + Result.yresize(byteOff + 32); + ui64 bitCode; + ui8 bitLength; + Compressor.GetStat(data, bitCode, bitLength); + ui64 dst; + memcpy(&dst, &Result[byteOff], sizeof(dst)); + ui64 mask = ((1ULL << (Offset & 7)) - 1ULL); + dst = (dst & mask) | (bitCode << (Offset & 7)); + memcpy(&Result[byteOff], &dst, sizeof(dst)); + Offset += bitLength; + } + ~THQCompressorIterator() { + Result.yresize((Offset + 7) >> 3); + if (Offset & 7) + Result.back() &= (1 << (Offset & 7)) - 1; + } + }; + template <bool HQ> + void Compress(const TStringBuf& stringBuf, TVector<char>& rslt) const { + if (!HQ) { + TCompressorIterator iterator(rslt, *this); + Iterate(stringBuf, iterator); } else { - THQCompressorIterator iterator(rslt, *this); - Iterate(stringBuf, iterator); + THQCompressorIterator iterator(rslt, *this); + Iterate(stringBuf, iterator); } } - }; - - class TDataDecompressor { - const TCompressorTable Table; - - public: - TDataDecompressor(const TCompressorTable& table) - : Table(table) + }; + + class TDataDecompressor { + const TCompressorTable Table; + + public: + TDataDecompressor(const TCompressorTable& table) + : Table(table) { } - size_t Decompress(ui32* data4, const ui8* dataBuff, ui32 type) const { - if (type == 0) { - data4[0] = 0; - return 0; - } - if (type == 1) { - ui8 masked; - memcpy(&masked, dataBuff, sizeof(masked)); - data4[0] = Table.Table[masked]; - return 1; - } else if (type == 2) { - ui16 masked; - memcpy(&masked, dataBuff, sizeof(masked)); - data4[0] = Table.Table[masked]; - return 2; - } else { - memcpy(data4, dataBuff, sizeof(*data4)); - return 4; - } - } - size_t Decompress(ui32* data4, const ui8* dataBuff) const { - ui32 status = *dataBuff; - const ui8* oldData = dataBuff; - ++dataBuff; - dataBuff += Decompress(data4 + 0, dataBuff, (status >> 6)); - dataBuff += Decompress(data4 + 1, dataBuff, (status >> 4) & 0x3); - dataBuff += Decompress(data4 + 2, dataBuff, (status >> 2) & 0x3); - dataBuff += Decompress(data4 + 3, dataBuff, (status)&0x3); - return dataBuff - oldData; + size_t Decompress(ui32* data4, const ui8* dataBuff, ui32 type) const { + if (type == 0) { + data4[0] = 0; + return 0; + } + if (type == 1) { + ui8 masked; + memcpy(&masked, dataBuff, sizeof(masked)); + data4[0] = Table.Table[masked]; + return 1; + } else if (type == 2) { + ui16 masked; + memcpy(&masked, dataBuff, sizeof(masked)); + data4[0] = Table.Table[masked]; + return 2; + } else { + memcpy(data4, dataBuff, sizeof(*data4)); + return 4; + } } - void Decompress(ui32* data, const ui8* dataBuff, ui64& offset, ui64 border) const { - size_t off = offset >> 3; - ui64 val = 0; - if (border - off >= sizeof(val)) { - memcpy(&val, dataBuff + off, sizeof(val)); - } else { - memcpy(&val, dataBuff + off, border - off); + size_t Decompress(ui32* data4, const ui8* dataBuff) const { + ui32 status = *dataBuff; + const ui8* oldData = dataBuff; + ++dataBuff; + dataBuff += Decompress(data4 + 0, dataBuff, (status >> 6)); + dataBuff += Decompress(data4 + 1, dataBuff, (status >> 4) & 0x3); + dataBuff += Decompress(data4 + 2, dataBuff, (status >> 2) & 0x3); + dataBuff += Decompress(data4 + 3, dataBuff, (status)&0x3); + return dataBuff - oldData; + } + void Decompress(ui32* data, const ui8* dataBuff, ui64& offset, ui64 border) const { + size_t off = offset >> 3; + ui64 val = 0; + if (border - off >= sizeof(val)) { + memcpy(&val, dataBuff + off, sizeof(val)); + } else { + memcpy(&val, dataBuff + off, border - off); } - val >>= (offset & 7); - ui8 index = Table.HuffIndices[ui8(val)]; - const NCompProto::TCoderEntry& entry = Table.HuffCodes[index]; - ui8 allBits = entry.AllBits; - val = (val & ((1ULL << allBits) - 1ULL)) >> entry.PrefixBits; - val = val + entry.MinValue; - data[0] = (index == 9) ? val : Table.Table[val]; - offset += allBits; + val >>= (offset & 7); + ui8 index = Table.HuffIndices[ui8(val)]; + const NCompProto::TCoderEntry& entry = Table.HuffCodes[index]; + ui8 allBits = entry.AllBits; + val = (val & ((1ULL << allBits) - 1ULL)) >> entry.PrefixBits; + val = val + entry.MinValue; + data[0] = (index == 9) ? val : Table.Table[val]; + offset += allBits; } - size_t GetJunkSize() const { - return 8; - } - template <bool HQ> - void Decompress(const TStringBuf& dataBuf, TVector<char>& rslt) const { - rslt.clear(); - if (dataBuf.empty()) { - return; - } + size_t GetJunkSize() const { + return 8; + } + template <bool HQ> + void Decompress(const TStringBuf& dataBuf, TVector<char>& rslt) const { + rslt.clear(); + if (dataBuf.empty()) { + return; + } const ui8* src = reinterpret_cast<const ui8*>(dataBuf.data()); ui64 border = dataBuf.size(); - ui32 len = 0; - ui32 nullTerm = 1; - if (HQ) { - ui64 offset = 0; - ui32 length = 0; - Decompress(&length, src, offset, border); - size_t length32 = (length + 3) / 4; - rslt.yresize(length32 * 4 + nullTerm); - ui32* result = reinterpret_cast<ui32*>(rslt.data()); - len = length; - for (size_t i = 0; i < length32; ++i) { - Decompress(&result[i], src, offset, border); - } - } else { - ui32 data[4]; - src += Decompress(data, src); - len = data[0]; - size_t length32x4 = (4 + len + 15) / 16; - rslt.yresize(length32x4 * 16 + nullTerm); - ui32* result = reinterpret_cast<ui32*>(rslt.data()); - for (size_t i = 0; i < 3; ++i) { - result[i] = data[i + 1]; - } - for (size_t j = 1; j < length32x4; ++j) { - src += Decompress(&result[j * 4 - 1], src); + ui32 len = 0; + ui32 nullTerm = 1; + if (HQ) { + ui64 offset = 0; + ui32 length = 0; + Decompress(&length, src, offset, border); + size_t length32 = (length + 3) / 4; + rslt.yresize(length32 * 4 + nullTerm); + ui32* result = reinterpret_cast<ui32*>(rslt.data()); + len = length; + for (size_t i = 0; i < length32; ++i) { + Decompress(&result[i], src, offset, border); } + } else { + ui32 data[4]; + src += Decompress(data, src); + len = data[0]; + size_t length32x4 = (4 + len + 15) / 16; + rslt.yresize(length32x4 * 16 + nullTerm); + ui32* result = reinterpret_cast<ui32*>(rslt.data()); + for (size_t i = 0; i < 3; ++i) { + result[i] = data[i + 1]; + } + for (size_t j = 1; j < length32x4; ++j) { + src += Decompress(&result[j * 4 - 1], src); + } } - rslt.resize(len); + rslt.resize(len); } }; - - struct TSamplerIterator { - TDataSampler& Sampler; - TSamplerIterator(TDataSampler& sampler) - : Sampler(sampler) + + struct TSamplerIterator { + TDataSampler& Sampler; + TSamplerIterator(TDataSampler& sampler) + : Sampler(sampler) { } void Visit(const ui32 data) { - Sampler.AddStat(data); + Sampler.AddStat(data); } - }; - - struct TGreaterComparer { - //std::greater in arcadia??? - bool operator()(ui64 a, ui64 b) const { - return a > b; + }; + + struct TGreaterComparer { + //std::greater in arcadia??? + bool operator()(ui64 a, ui64 b) const { + return a > b; } }; - - TDataSampler::TDataSampler() { - memset(this, 0, sizeof(*this)); + + TDataSampler::TDataSampler() { + memset(this, 0, sizeof(*this)); } - void TDataSampler::BuildTable(TCompressorTable& table) const { - std::vector<ui64> sorted; - for (size_t i = 0; i < Size; ++i) { - ui64 res = (ui64(EntryHit[i]) << 32) + ui32(i); - sorted.push_back(res); + void TDataSampler::BuildTable(TCompressorTable& table) const { + std::vector<ui64> sorted; + for (size_t i = 0; i < Size; ++i) { + ui64 res = (ui64(EntryHit[i]) << 32) + ui32(i); + sorted.push_back(res); } - std::vector<ui32> entryTbl(Size); - std::sort(sorted.begin(), sorted.end(), TGreaterComparer()); - table.HashMul = magicHashMul; - i64 freqs[65536]; - for (size_t i = 0; i < 65536; ++i) { - ui32 ind = ui32(sorted[i]); - table.Table[i] = EntryVal[ind]; - freqs[i] = EntryHit[ind]; + std::vector<ui32> entryTbl(Size); + std::sort(sorted.begin(), sorted.end(), TGreaterComparer()); + table.HashMul = magicHashMul; + i64 freqs[65536]; + for (size_t i = 0; i < 65536; ++i) { + ui32 ind = ui32(sorted[i]); + table.Table[i] = EntryVal[ind]; + freqs[i] = EntryHit[ind]; } - table.BuildHuffCodes(Counter, freqs); + table.BuildHuffCodes(Counter, freqs); } - - void TDataSampler::AddStat(ui32 val) { - ++Counter; - size_t hashInd = HashIndex(val, magicHashMul); - if (EntryVal[hashInd] == val) { - ++EntryHit[hashInd]; - } else if (EntryHit[hashInd] > 1) { - --EntryHit[hashInd]; + + void TDataSampler::AddStat(ui32 val) { + ++Counter; + size_t hashInd = HashIndex(val, magicHashMul); + if (EntryVal[hashInd] == val) { + ++EntryHit[hashInd]; + } else if (EntryHit[hashInd] > 1) { + --EntryHit[hashInd]; } else { - EntryHit[hashInd] = 1; - EntryVal[hashInd] = val; + EntryHit[hashInd] = 1; + EntryVal[hashInd] = val; } } - - void TDataSampler::AddStat(const TStringBuf& stringBuf) { - TSamplerIterator iterator(*this); - Iterate(stringBuf, iterator); + + void TDataSampler::AddStat(const TStringBuf& stringBuf) { + TSamplerIterator iterator(*this); + Iterate(stringBuf, iterator); } - TChunkCompressor::TChunkCompressor(bool highQuality, const TCompressorTable& table) - : HighQuality(highQuality) + TChunkCompressor::TChunkCompressor(bool highQuality, const TCompressorTable& table) + : HighQuality(highQuality) { - Compressor.Reset(new TDataCompressor(table)); + Compressor.Reset(new TDataCompressor(table)); } - void TChunkCompressor::Compress(TStringBuf data, TVector<char>* rslt) const { - if (HighQuality) { - Compressor->Compress<1>(data, *rslt); - } else { - Compressor->Compress<0>(data, *rslt); - } + void TChunkCompressor::Compress(TStringBuf data, TVector<char>* rslt) const { + if (HighQuality) { + Compressor->Compress<1>(data, *rslt); + } else { + Compressor->Compress<0>(data, *rslt); + } } - TChunkCompressor::~TChunkCompressor() = default; + TChunkCompressor::~TChunkCompressor() = default; - TChunkDecompressor::TChunkDecompressor(bool highQuality, const TCompressorTable& table) - : HighQuality(highQuality) - { - Decompressor.Reset(new TDataDecompressor(table)); + TChunkDecompressor::TChunkDecompressor(bool highQuality, const TCompressorTable& table) + : HighQuality(highQuality) + { + Decompressor.Reset(new TDataDecompressor(table)); } - void TChunkDecompressor::Decompress(TStringBuf data, TVector<char>* rslt) const { - if (HighQuality) { - Decompressor->Decompress<1>(data, *rslt); - } else { - Decompressor->Decompress<0>(data, *rslt); - } + void TChunkDecompressor::Decompress(TStringBuf data, TVector<char>* rslt) const { + if (HighQuality) { + Decompressor->Decompress<1>(data, *rslt); + } else { + Decompressor->Decompress<0>(data, *rslt); + } } - TChunkDecompressor::~TChunkDecompressor() = default; + TChunkDecompressor::~TChunkDecompressor() = default; } diff --git a/library/cpp/comptable/comptable.h b/library/cpp/comptable/comptable.h index d225fed7a0..ff9f241964 100644 --- a/library/cpp/comptable/comptable.h +++ b/library/cpp/comptable/comptable.h @@ -1,5 +1,5 @@ #pragma once - + #include <util/generic/vector.h> #include <util/memory/blob.h> #include <util/ysaveload.h> @@ -7,62 +7,62 @@ #include <library/cpp/compproto/huff.h> namespace NCompTable { - struct TCompressorTable { - ui32 Table[65536]; - ui32 HashMul; - NCompProto::TCoderEntry HuffCodes[10]; - ui8 HuffIndices[256]; + struct TCompressorTable { + ui32 Table[65536]; + ui32 HashMul; + NCompProto::TCoderEntry HuffCodes[10]; + ui8 HuffIndices[256]; - void GetHuffCode(const NCompProto::TCoderEntry& entry, ui32 value, ui64& bitCode, ui8& bitLength) const; - void GetLastHuffCode(ui32 value, ui64& bitCode, ui8& bitLength) const; - void GetHuffCode(ui32 value, ui64& bitCode, ui8& bitLength) const; - ui8 GetHuffIndex(ui8 prefix); - void BuildHuffCodes(i64 totalFreq, i64 freqs[65536]); - bool BuildHuffCodes(i64 totalFreq, i64 freqs[65536], i64 add); - }; + void GetHuffCode(const NCompProto::TCoderEntry& entry, ui32 value, ui64& bitCode, ui8& bitLength) const; + void GetLastHuffCode(ui32 value, ui64& bitCode, ui8& bitLength) const; + void GetHuffCode(ui32 value, ui64& bitCode, ui8& bitLength) const; + ui8 GetHuffIndex(ui8 prefix); + void BuildHuffCodes(i64 totalFreq, i64 freqs[65536]); + bool BuildHuffCodes(i64 totalFreq, i64 freqs[65536], i64 add); + }; - struct TDataSampler { - enum { - Size = 1 << 18, - }; - ui32 EntryVal[Size]; - i64 EntryHit[Size]; - i64 Counter; + struct TDataSampler { + enum { + Size = 1 << 18, + }; + ui32 EntryVal[Size]; + i64 EntryHit[Size]; + i64 Counter; - public: - TDataSampler(); - void BuildTable(TCompressorTable& table) const; - void AddStat(ui32 val); - void AddStat(const TStringBuf& stringBuf); + public: + TDataSampler(); + void BuildTable(TCompressorTable& table) const; + void AddStat(ui32 val); + void AddStat(const TStringBuf& stringBuf); }; - class TDataCompressor; - class TDataDecompressor; + class TDataCompressor; + class TDataDecompressor; - class TChunkCompressor { - public: - TChunkCompressor(bool highQuality, const TCompressorTable& table); - void Compress(TStringBuf data, TVector<char>* result) const; - ~TChunkCompressor(); + class TChunkCompressor { + public: + TChunkCompressor(bool highQuality, const TCompressorTable& table); + void Compress(TStringBuf data, TVector<char>* result) const; + ~TChunkCompressor(); - private: - bool HighQuality; - THolder<TDataCompressor> Compressor; - }; + private: + bool HighQuality; + THolder<TDataCompressor> Compressor; + }; - class TChunkDecompressor { - public: - TChunkDecompressor(bool highQuality, const TCompressorTable& table); - void Decompress(TStringBuf data, TVector<char>* result) const; - ~TChunkDecompressor(); - - private: - bool HighQuality; - THolder<TDataDecompressor> Decompressor; - }; + class TChunkDecompressor { + public: + TChunkDecompressor(bool highQuality, const TCompressorTable& table); + void Decompress(TStringBuf data, TVector<char>* result) const; + ~TChunkDecompressor(); -} + private: + bool HighQuality; + THolder<TDataDecompressor> Decompressor; + }; +} + template <> class TSerializer<NCompTable::TCompressorTable> { public: diff --git a/library/cpp/comptable/usage/usage.cpp b/library/cpp/comptable/usage/usage.cpp index 9997c83686..550b6777c3 100644 --- a/library/cpp/comptable/usage/usage.cpp +++ b/library/cpp/comptable/usage/usage.cpp @@ -1,15 +1,15 @@ #include <library/cpp/comptable/comptable.h> -#include <util/random/random.h> -#include <util/random/fast.h> - +#include <util/random/random.h> +#include <util/random/fast.h> + #include <time.h> #include <stdlib.h> using namespace NCompTable; -template <bool HQ> -void DoTest(const TCompressorTable& table, const TVector<TString>& lines) { +template <bool HQ> +void DoTest(const TCompressorTable& table, const TVector<TString>& lines) { TVector<char> compressed; TVector<char> decompressed; @@ -19,8 +19,8 @@ void DoTest(const TCompressorTable& table, const TVector<TString>& lines) { size_t origSize = 0; size_t compSize = 0; float cl1 = clock(); - for (size_t i = 0; i < lines.size(); ++i) { - const TString& line = lines[i]; + for (size_t i = 0; i < lines.size(); ++i) { + const TString& line = lines[i]; compressor.Compress(line, &compressed); origSize += line.size(); compSize += compressed.size(); @@ -28,20 +28,20 @@ void DoTest(const TCompressorTable& table, const TVector<TString>& lines) { deCompressor.Decompress(in, &decompressed); if (decompressed.size() != line.size() || memcmp(decompressed.data(), line.data(), decompressed.size())) { Cout << i << "\n"; - Cout << line << "\n" - << TString(decompressed.data(), decompressed.size()) << "\n"; + Cout << line << "\n" + << TString(decompressed.data(), decompressed.size()) << "\n"; abort(); } } float cl2 = clock(); float secs = (cl2 - cl1) / CLOCKS_PER_SEC; Cout << "origSize: " << origSize << "\tcompSize: " << compSize << Endl; - Cout << "yep! compression + decompression speed " << origSize / 1024.0f / 1024.0f / secs << " mbps\n"; + Cout << "yep! compression + decompression speed " << origSize / 1024.0f / 1024.0f / secs << " mbps\n"; Cout << "yep! compression ratio " << double(origSize) / double(compSize + 1) << "\n"; } -int main(int argc, const char* argv[]) { - TReallyFastRng32 rr(17); +int main(int argc, const char* argv[]) { + TReallyFastRng32 rr(17); TVector<TString> lines; /*FILE *fp = fopen("res", "rb"); while (!feof(fp)) { @@ -52,16 +52,16 @@ int main(int argc, const char* argv[]) { //for (size_t i = 0; i < 10000000; ++i) { //for (size_t i = 0; i < 1000000; ++i) { for (size_t i = 0; i < 1000000; ++i) { - size_t size = rr.Uniform(32); + size_t size = rr.Uniform(32); TString res = "www.yandex.ru/yandsearch?text="; for (size_t j = 0; j < size; ++j) { - res += "qwer"[rr.Uniform(4)]; + res += "qwer"[rr.Uniform(4)]; } lines.push_back(res); } THolder<TDataSampler> sampler(new TDataSampler); for (size_t i = 0; i < lines.size(); ++i) { - sampler->AddStat(lines[i]); + sampler->AddStat(lines[i]); } TCompressorTable table; sampler->BuildTable(table); diff --git a/library/cpp/comptable/ut/comptable_ut.cpp b/library/cpp/comptable/ut/comptable_ut.cpp index 5901d0246f..5bb2058fce 100644 --- a/library/cpp/comptable/ut/comptable_ut.cpp +++ b/library/cpp/comptable/ut/comptable_ut.cpp @@ -1,13 +1,13 @@ #include <library/cpp/comptable/comptable.h> #include <library/cpp/testing/unittest/registar.h> -#include <util/random/random.h> -#include <util/random/fast.h> - +#include <util/random/random.h> +#include <util/random/fast.h> + using namespace NCompTable; -template <bool HQ> -void DoTest(const TCompressorTable& table, const TVector<TString>& lines) { +template <bool HQ> +void DoTest(const TCompressorTable& table, const TVector<TString>& lines) { TVector<char> compressed; TVector<char> decompressed; @@ -21,8 +21,8 @@ void DoTest(const TCompressorTable& table, const TVector<TString>& lines) { size_t origSize = 0; size_t compSize = 0; - for (size_t i = 0; i < lines.size(); ++i) { - const TString& line = lines[i]; + for (size_t i = 0; i < lines.size(); ++i) { + const TString& line = lines[i]; compressor.Compress(line, &compressed); origSize += line.size(); compSize += compressed.size(); @@ -30,30 +30,30 @@ void DoTest(const TCompressorTable& table, const TVector<TString>& lines) { deCompressor.Decompress(in, &decompressed); UNIT_ASSERT(decompressed.size() == line.size() && memcmp(decompressed.data(), line.data(), decompressed.size()) == 0); } - UNIT_ASSERT_EQUAL(origSize, 45491584); + UNIT_ASSERT_EQUAL(origSize, 45491584); if (HQ) { - UNIT_ASSERT_EQUAL(compSize, 11074583); + UNIT_ASSERT_EQUAL(compSize, 11074583); } else { - UNIT_ASSERT_EQUAL(compSize, 17459336); + UNIT_ASSERT_EQUAL(compSize, 17459336); } UNIT_ASSERT(compSize < origSize); } Y_UNIT_TEST_SUITE(TestComptable) { Y_UNIT_TEST(TestComptableCompressDecompress) { - TReallyFastRng32 rr(17); + TReallyFastRng32 rr(17); TVector<TString> lines; for (size_t i = 0; i < 1000000; ++i) { - size_t size = rr.Uniform(32); + size_t size = rr.Uniform(32); TString res = "www.yandex.ru/yandsearch?text="; for (size_t j = 0; j < size; ++j) { - res += "qwer"[rr.Uniform(4)]; + res += "qwer"[rr.Uniform(4)]; } lines.push_back(res); } THolder<TDataSampler> sampler(new TDataSampler); for (size_t i = 0; i < lines.size(); ++i) { - sampler->AddStat(lines[i]); + sampler->AddStat(lines[i]); } TCompressorTable table; sampler->BuildTable(table); diff --git a/library/cpp/containers/2d_array/2d_array.h b/library/cpp/containers/2d_array/2d_array.h index 9e24650637..d57f6e4b2f 100644 --- a/library/cpp/containers/2d_array/2d_array.h +++ b/library/cpp/containers/2d_array/2d_array.h @@ -1,5 +1,5 @@ #pragma once - + #include <util/system/yassert.h> #include <util/generic/algorithm.h> @@ -27,9 +27,9 @@ private: T** PData; size_t XSize; size_t YSize; - + private: - void Copy(const TArray2D& a) { + void Copy(const TArray2D& a) { XSize = a.XSize; YSize = a.YSize; Create(); @@ -46,17 +46,17 @@ private: for (size_t i = 0; i < YSize; i++) PData[i] = Data + i * XSize; } - + public: TArray2D(size_t xsize = 1, size_t ysize = 1) { XSize = xsize; YSize = ysize; Create(); } - TArray2D(const TArray2D& a) { + TArray2D(const TArray2D& a) { Copy(a); } - TArray2D& operator=(const TArray2D& a) { + TArray2D& operator=(const TArray2D& a) { Destroy(); Copy(a); return *this; @@ -73,7 +73,7 @@ public: Create(); } void Clear() { - SetSizes(1, 1); + SetSizes(1, 1); } #ifdef _DEBUG TBoundCheck<T> operator[](size_t i) const { @@ -107,8 +107,8 @@ public: } }; -template <class T> -inline bool operator==(const TArray2D<T>& a, const TArray2D<T>& b) { +template <class T> +inline bool operator==(const TArray2D<T>& a, const TArray2D<T>& b) { if (a.GetXSize() != b.GetXSize() || a.GetYSize() != b.GetYSize()) return false; for (size_t y = 0; y < a.GetYSize(); ++y) { @@ -119,7 +119,7 @@ inline bool operator==(const TArray2D<T>& a, const TArray2D<T>& b) { return true; } -template <class T> -inline bool operator!=(const TArray2D<T>& a, const TArray2D<T>& b) { +template <class T> +inline bool operator!=(const TArray2D<T>& a, const TArray2D<T>& b) { return !(a == b); } diff --git a/library/cpp/containers/2d_array/ya.make b/library/cpp/containers/2d_array/ya.make index 71d56b902f..c542a46070 100644 --- a/library/cpp/containers/2d_array/ya.make +++ b/library/cpp/containers/2d_array/ya.make @@ -1,6 +1,6 @@ LIBRARY() -OWNER(kirillovs) +OWNER(kirillovs) SRCS( 2d_array.cpp diff --git a/library/cpp/containers/atomizer/atomizer.cpp b/library/cpp/containers/atomizer/atomizer.cpp index 7a5f781d99..90d526adb4 100644 --- a/library/cpp/containers/atomizer/atomizer.cpp +++ b/library/cpp/containers/atomizer/atomizer.cpp @@ -1 +1 @@ -#include "atomizer.h" +#include "atomizer.h" diff --git a/library/cpp/containers/atomizer/atomizer.h b/library/cpp/containers/atomizer/atomizer.h index 5e40f47ab9..8f9e5c5001 100644 --- a/library/cpp/containers/atomizer/atomizer.h +++ b/library/cpp/containers/atomizer/atomizer.h @@ -1,24 +1,24 @@ #pragma once #include <library/cpp/containers/str_map/str_map.h> - + #include <util/generic/vector.h> #include <util/generic/utility.h> - + #include <utility> -#include <cstdio> - -template <class HashFcn = THash<const char*>, class EqualTo = TEqualTo<const char*>> -class atomizer; - -template <class T, class HashFcn = THash<const char*>, class EqualTo = TEqualTo<const char*>> -class super_atomizer; - +#include <cstdio> + +template <class HashFcn = THash<const char*>, class EqualTo = TEqualTo<const char*>> +class atomizer; + +template <class T, class HashFcn = THash<const char*>, class EqualTo = TEqualTo<const char*>> +class super_atomizer; + template <class HashFcn, class EqualTo> -class atomizer: public string_hash<ui32, HashFcn, EqualTo> { +class atomizer: public string_hash<ui32, HashFcn, EqualTo> { private: TVector<const char*> order; - + public: using iterator = typename string_hash<ui32, HashFcn, EqualTo>::iterator; using const_iterator = typename string_hash<ui32, HashFcn, EqualTo>::const_iterator; @@ -33,84 +33,84 @@ public: using string_hash<ui32, HashFcn, EqualTo>::insert_copy; using string_hash<ui32, HashFcn, EqualTo>::clear_hash; - atomizer() { + atomizer() { order.reserve(HASH_SIZE_DEFAULT); } atomizer(size_type hash_size, pool_size_type pool_size) - : string_hash<ui32, HashFcn, EqualTo>(hash_size, pool_size) + : string_hash<ui32, HashFcn, EqualTo>(hash_size, pool_size) { order.reserve(hash_size); } ~atomizer() = default; - ui32 string_to_atom(const char* key) { - const char* old_begin = pool.Begin(); - const char* old_end = pool.End(); + ui32 string_to_atom(const char* key) { + const char* old_begin = pool.Begin(); + const char* old_end = pool.End(); std::pair<iterator, bool> ins = insert_copy(key, ui32(size() + 1)); - if (ins.second) { // new? - if (pool.Begin() != old_begin) // repoint? + if (ins.second) { // new? + if (pool.Begin() != old_begin) // repoint? for (TVector<const char*>::iterator ptr = order.begin(); ptr != order.end(); ++ptr) if (old_begin <= *ptr && *ptr < old_end) // from old pool? - *ptr += pool.Begin() - old_begin; + *ptr += pool.Begin() - old_begin; order.push_back((*ins.first).first); // copy of 'key' } return (ui32)(*ins.first).second; } - ui32 perm_string_to_atom(const char* key) { - value_type val(key, ui32(size() + 1)); + ui32 perm_string_to_atom(const char* key) { + value_type val(key, ui32(size() + 1)); std::pair<iterator, bool> ins = this->insert(val); if (ins.second) order.push_back((*ins.first).first); // == copy of 'key' - return (ui32)(*ins.first).second; // == size()+1 + return (ui32)(*ins.first).second; // == size()+1 } - ui32 find_atom(const char* key) const { + ui32 find_atom(const char* key) const { const_iterator it = find(key); if (it == end()) return 0; // INVALID_ATOM else return (ui32)(*it).second; } - const char* get_atom_name(ui32 atom) const { + const char* get_atom_name(ui32 atom) const { if (atom && atom <= size()) - return order[atom - 1]; + return order[atom - 1]; return nullptr; } - void clear_atomizer() { + void clear_atomizer() { clear_hash(); order.clear(); } - void SaveC2N(FILE* f) const { // we write sorted file + void SaveC2N(FILE* f) const { // we write sorted file for (ui32 i = 0; i < order.size(); i++) if (order[i]) - fprintf(f, "%d\t%s\n", i + 1, order[i]); + fprintf(f, "%d\t%s\n", i + 1, order[i]); } - void LoadC2N(FILE* f) { // but can read unsorted one + void LoadC2N(FILE* f) { // but can read unsorted one long k, km = 0; char buf[1000]; - char* s; + char* s; while (fgets(buf, 1000, f)) { k = strtol(buf, &s, 10); - char* endl = strchr(s, '\n'); + char* endl = strchr(s, '\n'); if (endl) *endl = 0; if (k > 0 && k != LONG_MAX) { - km = Max(km, k); + km = Max(km, k); insert_copy(++s, ui32(k)); } } order.resize(km); - memset(&order[0], 0, order.size()); // if some atoms are absent + memset(&order[0], 0, order.size()); // if some atoms are absent for (const_iterator I = this->begin(); I != end(); ++I) order[(*I).second - 1] = (*I).first; } }; template <class T, class HashFcn, class EqualTo> -class super_atomizer: public string_hash<ui32, HashFcn, EqualTo> { +class super_atomizer: public string_hash<ui32, HashFcn, EqualTo> { private: using TOrder = TVector<std::pair<const char*, T>>; TOrder order; - + public: using iterator = typename string_hash<ui32, HashFcn, EqualTo>::iterator; using const_iterator = typename string_hash<ui32, HashFcn, EqualTo>::const_iterator; @@ -129,56 +129,56 @@ public: using string_hash<ui32, HashFcn, EqualTo>::insert_copy; using string_hash<ui32, HashFcn, EqualTo>::clear_hash; - super_atomizer() { + super_atomizer() { order.reserve(HASH_SIZE_DEFAULT); } super_atomizer(size_type hash_size, pool_size_type pool_size) - : string_hash<ui32, HashFcn, EqualTo>(hash_size, pool_size) + : string_hash<ui32, HashFcn, EqualTo>(hash_size, pool_size) { order.reserve(hash_size); } ~super_atomizer() = default; - ui32 string_to_atom(const char* key, const T* atom_data = NULL) { - const char* old_begin = pool.Begin(); - const char* old_end = pool.End(); + ui32 string_to_atom(const char* key, const T* atom_data = NULL) { + const char* old_begin = pool.Begin(); + const char* old_end = pool.End(); std::pair<iterator, bool> ins = insert_copy(key, ui32(size() + 1)); - if (ins.second) { // new? - if (pool.Begin() != old_begin) // repoint? + if (ins.second) { // new? + if (pool.Begin() != old_begin) // repoint? for (typename TOrder::iterator ptr = order.begin(); ptr != order.end(); ++ptr) if (old_begin <= (*ptr).first && (*ptr).first < old_end) // from old pool? - (*ptr).first += pool.Begin() - old_begin; + (*ptr).first += pool.Begin() - old_begin; order.push_back(std::pair<const char*, T>((*ins.first).first, atom_data ? *atom_data : T())); } return (*ins.first).second; } - ui32 perm_string_to_atom(const char* key, const T* atom_data = NULL) { - value_type val(key, ui32(size() + 1)); + ui32 perm_string_to_atom(const char* key, const T* atom_data = NULL) { + value_type val(key, ui32(size() + 1)); std::pair<iterator, bool> ins = this->insert(val); if (ins.second) order.push_back(std::pair<const char*, T>((*ins.first).first, atom_data ? *atom_data : T())); return (*ins.first).second; // == size()+1 } - ui32 find_atom(const char* key) const { + ui32 find_atom(const char* key) const { const_iterator it = find(key); if (it == end()) return 0; // INVALID_ATOM else return (*it).second; } - const char* get_atom_name(ui32 atom) const { + const char* get_atom_name(ui32 atom) const { if (atom && atom <= size()) - return order[atom - 1].first; + return order[atom - 1].first; return nullptr; } const T* get_atom_data(ui32 atom) const { if (atom && atom <= size()) - return &order[atom - 1].second; + return &order[atom - 1].second; return NULL; } T* get_atom_data(ui32 atom) { if (atom && atom <= size()) - return &order[atom - 1].second; + return &order[atom - 1].second; return NULL; } o_iterator o_begin() { @@ -193,7 +193,7 @@ public: o_const_iterator o_end() const { return order.end(); } - void clear_atomizer() { + void clear_atomizer() { clear_hash(); order.clear(); } diff --git a/library/cpp/containers/atomizer/ya.make b/library/cpp/containers/atomizer/ya.make index 55165a3b67..c3ad49fff2 100644 --- a/library/cpp/containers/atomizer/ya.make +++ b/library/cpp/containers/atomizer/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - +LIBRARY() + OWNER(g:util) - -PEERDIR( + +PEERDIR( library/cpp/containers/str_map -) - -SRCS( - atomizer.cpp -) - -END() +) + +SRCS( + atomizer.cpp +) + +END() diff --git a/library/cpp/containers/bitseq/bititerator.h b/library/cpp/containers/bitseq/bititerator.h index 52dadd3798..6eeaec4073 100644 --- a/library/cpp/containers/bitseq/bititerator.h +++ b/library/cpp/containers/bitseq/bititerator.h @@ -3,7 +3,7 @@ #include "traits.h" #include <library/cpp/pop_count/popcount.h> - + template <typename T> class TBitIterator { public: @@ -15,8 +15,8 @@ public: : Current(0) , Mask(0) , Data(data) - { - } + { + } /// Get the word next to the one we are currenlty iterating over. const TWord* NextWord() const { @@ -49,7 +49,7 @@ public: if (!Mask) return *Data & TTraits::ElemMask(count); - auto usedBits = (size_t)PopCount(Mask - 1); + auto usedBits = (size_t)PopCount(Mask - 1); TWord result = Current >> usedBits; auto leftInCurrent = TTraits::NumBits - usedBits; if (count <= leftInCurrent) @@ -72,7 +72,7 @@ public: return Current & TTraits::ElemMask(count); } - auto usedBits = (size_t)PopCount(Mask - 1); + auto usedBits = (size_t)PopCount(Mask - 1); TWord result = Current >> usedBits; auto leftInCurrent = TTraits::NumBits - usedBits; if (count < leftInCurrent) { @@ -97,7 +97,7 @@ public: if (!count) return; - int leftInCurrent = (size_t)PopCount(~(Mask - 1)); + int leftInCurrent = (size_t)PopCount(~(Mask - 1)); if (count < leftInCurrent) { Mask <<= count; return; diff --git a/library/cpp/containers/bitseq/bititerator_ut.cpp b/library/cpp/containers/bitseq/bititerator_ut.cpp index ed0925866f..e5aa5a9184 100644 --- a/library/cpp/containers/bitseq/bititerator_ut.cpp +++ b/library/cpp/containers/bitseq/bititerator_ut.cpp @@ -12,7 +12,7 @@ Y_UNIT_TEST_SUITE(TBitIteratorTest) { } template <typename TWord> - void AssertPeekRead(TBitIterator<TWord> & iter, ui8 count, TWord expected) { + void AssertPeekRead(TBitIterator<TWord> & iter, ui8 count, TWord expected) { auto peek = iter.Peek(count); auto read = iter.Read(count); UNIT_ASSERT_EQUAL(peek, read); diff --git a/library/cpp/containers/bitseq/bitvector.cpp b/library/cpp/containers/bitseq/bitvector.cpp index 05cb3a881d..895baeb99b 100644 --- a/library/cpp/containers/bitseq/bitvector.cpp +++ b/library/cpp/containers/bitseq/bitvector.cpp @@ -1 +1 @@ -#include "bitvector.h" +#include "bitvector.h" diff --git a/library/cpp/containers/bitseq/bitvector.h b/library/cpp/containers/bitseq/bitvector.h index 3f8fd81ee5..e5d171a203 100644 --- a/library/cpp/containers/bitseq/bitvector.h +++ b/library/cpp/containers/bitseq/bitvector.h @@ -3,7 +3,7 @@ #include "traits.h" #include <library/cpp/pop_count/popcount.h> - + #include <util/generic/vector.h> #include <util/ysaveload.h> @@ -78,8 +78,8 @@ public: } void Set(ui64 pos, TWord value, ui8 width, TWord mask) { - if (!width) - return; + if (!width) + return; Y_ASSERT((pos + width) <= Size_); size_t word = pos >> TTraits::DivShift; TWord shift1 = pos & TTraits::ModMask; @@ -87,8 +87,8 @@ public: Data_[word] &= ~(mask << shift1); Data_[word] |= (value & mask) << shift1; if (shift2 < width) { - Data_[word + 1] &= ~(mask >> shift2); - Data_[word + 1] |= (value & mask) >> shift2; + Data_[word + 1] &= ~(mask >> shift2); + Data_[word + 1] |= (value & mask) >> shift2; } } @@ -97,8 +97,8 @@ public: } void Append(TWord value, ui8 width, TWord mask) { - if (!width) - return; + if (!width) + return; if (Data_.size() * TTraits::NumBits < Size_ + width) { Data_.push_back(0); } @@ -113,7 +113,7 @@ public: size_t Count() const { size_t count = 0; for (size_t i = 0; i < Data_.size(); ++i) { - count += (size_t)PopCount(Data_[i]); + count += (size_t)PopCount(Data_[i]); } return count; } @@ -142,7 +142,7 @@ public: ui64 Space() const { return CHAR_BIT * (sizeof(Size_) + - Data_.size() * sizeof(TWord)); + Data_.size() * sizeof(TWord)); } void Print(IOutputStream& out, size_t truncate = 128) { diff --git a/library/cpp/containers/bitseq/readonly_bitvector.h b/library/cpp/containers/bitseq/readonly_bitvector.h index 8612739c3f..419e49811e 100644 --- a/library/cpp/containers/bitseq/readonly_bitvector.h +++ b/library/cpp/containers/bitseq/readonly_bitvector.h @@ -16,8 +16,8 @@ public: TReadonlyBitVector() : Size_() , Data_() - { - } + { + } explicit TReadonlyBitVector(const TBitVector<T>& vector) : Size_(vector.Size_) diff --git a/library/cpp/containers/bitseq/ya.make b/library/cpp/containers/bitseq/ya.make index 7090956c55..758e2bec77 100644 --- a/library/cpp/containers/bitseq/ya.make +++ b/library/cpp/containers/bitseq/ya.make @@ -2,14 +2,14 @@ LIBRARY() OWNER(g:util) -PEERDIR( - util/draft +PEERDIR( + util/draft library/cpp/pop_count -) - -SRCS( - bitvector.cpp +) + +SRCS( + bitvector.cpp readonly_bitvector.cpp -) - +) + END() diff --git a/library/cpp/containers/compact_vector/compact_vector.h b/library/cpp/containers/compact_vector/compact_vector.h index dbe7473f0c..9e37938546 100644 --- a/library/cpp/containers/compact_vector/compact_vector.h +++ b/library/cpp/containers/compact_vector/compact_vector.h @@ -24,11 +24,11 @@ private: T* Ptr; THeader* Header() { - return ((THeader*)Ptr) - 1; + return ((THeader*)Ptr) - 1; } const THeader* Header() const { - return ((THeader*)Ptr) - 1; + return ((THeader*)Ptr) - 1; } public: @@ -40,8 +40,8 @@ public: TCompactVector() : Ptr(nullptr) - { - } + { + } TCompactVector(const TThis& that) : Ptr(nullptr) @@ -105,7 +105,7 @@ public: void* mem = ::malloc(sizeof(THeader) + newCapacity * sizeof(T)); if (mem == nullptr) ythrow yexception() << "out of memory"; - Ptr = (T*)(((THeader*)mem) + 1); + Ptr = (T*)(((THeader*)mem) + 1); Header()->Size = 0; Header()->Capacity = newCapacity; } else { @@ -197,12 +197,12 @@ public: Clear(); } - T& operator[](size_t index) { + T& operator[](size_t index) { Y_ASSERT(index < Size()); return Ptr[index]; } - const T& operator[](size_t index) const { + const T& operator[](size_t index) const { Y_ASSERT(index < Size()); return Ptr[index]; } diff --git a/library/cpp/containers/comptrie/array_with_size.h b/library/cpp/containers/comptrie/array_with_size.h index 36e61c7410..148200fe6a 100644 --- a/library/cpp/containers/comptrie/array_with_size.h +++ b/library/cpp/containers/comptrie/array_with_size.h @@ -3,19 +3,19 @@ #include <util/generic/ptr.h> #include <util/generic/noncopyable.h> #include <util/generic/utility.h> -#include <util/system/sys_alloc.h> +#include <util/system/sys_alloc.h> template <typename T> -class TArrayWithSizeHolder : TNonCopyable { +class TArrayWithSizeHolder : TNonCopyable { typedef TArrayWithSizeHolder<T> TThis; T* Data; public: - TArrayWithSizeHolder() - : Data(nullptr) - { - } + TArrayWithSizeHolder() + : Data(nullptr) + { + } ~TArrayWithSizeHolder() { if (!Data) @@ -26,7 +26,7 @@ public: } catch (...) { } } - y_deallocate(((size_t*)Data) - 1); + y_deallocate(((size_t*)Data) - 1); } void Swap(TThis& copy) { @@ -37,7 +37,7 @@ public: if (newSize == Size()) return; TThis copy; - copy.Data = (T*)(((size_t*)y_allocate(sizeof(size_t) + sizeof(T) * newSize)) + 1); + copy.Data = (T*)(((size_t*)y_allocate(sizeof(size_t) + sizeof(T) * newSize)) + 1); // does not handle constructor exceptions properly for (size_t i = 0; i < Min(Size(), newSize); ++i) { new (copy.Data + i) T(Data[i]); @@ -45,12 +45,12 @@ public: for (size_t i = Min(Size(), newSize); i < newSize; ++i) { new (copy.Data + i) T; } - ((size_t*)copy.Data)[-1] = newSize; + ((size_t*)copy.Data)[-1] = newSize; Swap(copy); } size_t Size() const { - return Data ? ((size_t*)Data)[-1] : 0; + return Data ? ((size_t*)Data)[-1] : 0; } bool Empty() const { diff --git a/library/cpp/containers/comptrie/chunked_helpers_trie.h b/library/cpp/containers/comptrie/chunked_helpers_trie.h index cfa35f5ba2..6efe55b5f0 100644 --- a/library/cpp/containers/comptrie/chunked_helpers_trie.h +++ b/library/cpp/containers/comptrie/chunked_helpers_trie.h @@ -13,7 +13,7 @@ public: : Trie(blob) { } - + bool Has(const char* key) const { return Trie.Find(key, strlen(key)); } @@ -23,7 +23,7 @@ public: } }; -template <bool sorted = false> +template <bool sorted = false> class TTrieSetWriter { private: TCompactTrieBuilder<char> Builder; @@ -57,24 +57,24 @@ public: } }; -template <bool isWriter, bool sorted = false> +template <bool isWriter, bool sorted = false> struct TTrieSetG; -template <bool sorted> +template <bool sorted> struct TTrieSetG<false, sorted> { typedef TTrieSet T; }; -template <bool sorted> +template <bool sorted> struct TTrieSetG<true, sorted> { typedef TTrieSetWriter<sorted> T; }; -template <typename T> +template <typename T> class TTrieMap { private: TCompactTrie<char> Trie; - static_assert(sizeof(T) <= sizeof(ui64), "expect sizeof(T) <= sizeof(ui64)"); + static_assert(sizeof(T) <= sizeof(ui64), "expect sizeof(T) <= sizeof(ui64)"); public: TTrieMap(const TBlob& blob) @@ -101,17 +101,17 @@ public: } } - const TCompactTrie<char>& GetTrie() const { + const TCompactTrie<char>& GetTrie() const { return Trie; } }; -template <typename T, bool sorted = false> +template <typename T, bool sorted = false> class TTrieMapWriter { private: typedef TCompactTrieBuilder<char> TBuilder; TBuilder Builder; - static_assert(sizeof(T) <= sizeof(ui64), "expect sizeof(T) <= sizeof(ui64)"); + static_assert(sizeof(T) <= sizeof(ui64), "expect sizeof(T) <= sizeof(ui64)"); #ifndef NDEBUG bool IsSorted; #endif @@ -130,12 +130,12 @@ public: memcpy(&intValue, &value, sizeof(T)); Builder.Add(key, strlen(key), intValue); #ifndef NDEBUG - /* + /* if (!IsSorted) { T test; assert(Get(key, &test) && value == test); } - */ + */ #endif } @@ -177,7 +177,7 @@ public: } }; -template <typename T> +template <typename T> class TTrieSortedMapWriter { private: typedef std::pair<TString, T> TValue; @@ -204,15 +204,15 @@ public: } }; -template <typename X, bool isWriter, bool sorted = false> +template <typename X, bool isWriter, bool sorted = false> struct TTrieMapG; -template <typename X, bool sorted> +template <typename X, bool sorted> struct TTrieMapG<X, false, sorted> { typedef TTrieMap<X> T; }; -template <typename X, bool sorted> +template <typename X, bool sorted> struct TTrieMapG<X, true, sorted> { typedef TTrieMapWriter<X, sorted> T; }; diff --git a/library/cpp/containers/comptrie/comptrie.cpp b/library/cpp/containers/comptrie/comptrie.cpp index 4556e5b571..86a006a231 100644 --- a/library/cpp/containers/comptrie/comptrie.cpp +++ b/library/cpp/containers/comptrie/comptrie.cpp @@ -1,8 +1,8 @@ -#include "comptrie_impl.h" -#include "comptrie.h" -#include "array_with_size.h" -#include "comptrie_trie.h" -#include "comptrie_builder.h" -#include "protopacker.h" -#include "set.h" -#include "chunked_helpers_trie.h" +#include "comptrie_impl.h" +#include "comptrie.h" +#include "array_with_size.h" +#include "comptrie_trie.h" +#include "comptrie_builder.h" +#include "protopacker.h" +#include "set.h" +#include "chunked_helpers_trie.h" diff --git a/library/cpp/containers/comptrie/comptrie_builder.h b/library/cpp/containers/comptrie/comptrie_builder.h index cf7d2e39a3..e0364eef9f 100644 --- a/library/cpp/containers/comptrie/comptrie_builder.h +++ b/library/cpp/containers/comptrie/comptrie_builder.h @@ -26,18 +26,18 @@ enum ECompactTrieBuilderFlags { using TCompactTrieBuilderFlags = ECompactTrieBuilderFlags; -inline TCompactTrieBuilderFlags operator|(TCompactTrieBuilderFlags first, TCompactTrieBuilderFlags second) { +inline TCompactTrieBuilderFlags operator|(TCompactTrieBuilderFlags first, TCompactTrieBuilderFlags second) { return static_cast<TCompactTrieBuilderFlags>(static_cast<int>(first) | second); } -inline TCompactTrieBuilderFlags& operator|=(TCompactTrieBuilderFlags& first, TCompactTrieBuilderFlags second) { +inline TCompactTrieBuilderFlags& operator|=(TCompactTrieBuilderFlags& first, TCompactTrieBuilderFlags second) { return first = first | second; } template <typename T> class TArrayWithSizeHolder; -template <class T = char, class D = ui64, class S = TCompactTriePacker<D>> +template <class T = char, class D = ui64, class S = TCompactTriePacker<D>> class TCompactTrieBuilder { public: typedef T TSymbol; @@ -92,12 +92,12 @@ public: size_t GetEntryCount() const; size_t GetNodeCount() const; - + // Exact output file size in bytes. size_t MeasureByteSize() const { return Impl->MeasureByteSize(); } - + protected: class TCompactTrieBuilderImpl; THolder<TCompactTrieBuilderImpl> Impl; diff --git a/library/cpp/containers/comptrie/comptrie_builder.inl b/library/cpp/containers/comptrie/comptrie_builder.inl index f273fa6571..e19f005d7c 100644 --- a/library/cpp/containers/comptrie/comptrie_builder.inl +++ b/library/cpp/containers/comptrie/comptrie_builder.inl @@ -27,7 +27,7 @@ template <class T, class D, class S> class TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl { protected: - TMemoryPool Pool; + TMemoryPool Pool; size_t PayloadSize; THolder<TFixedSizeAllocator> NodeAllocator; class TNode; @@ -138,7 +138,7 @@ public: Y_ASSERT(reinterpret_cast<ISubtree*>(this) == static_cast<void*>(this)); // This assumption is used in TNode::Subtree() } - iterator Find(char ch); + iterator Find(char ch); const_iterator Find(char ch) const; void Add(const TBlob& s, TNode* node); @@ -422,8 +422,8 @@ public: template <class T, class D, class S> TCompactTrieBuilder<T, D, S>::TCompactTrieBuilder(TCompactTrieBuilderFlags flags, TPacker packer, IAllocator* alloc) : Impl(new TCompactTrieBuilderImpl(flags, packer, alloc)) -{ -} +{ +} template <class T, class D, class S> bool TCompactTrieBuilder<T, D, S>::Add(const TSymbol* key, size_t keylen, const TData& value) { @@ -742,7 +742,7 @@ bool TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::FindLongestPrefixImp template <class T, class D, class S> void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::Clear() { DestroyNode(Root); - Pool.Clear(); + Pool.Clear(); NodeAllocator.Reset(new TFixedSizeAllocator(sizeof(TNode) + PayloadSize, TDefaultAllocator::Instance())); Root = new (*NodeAllocator) TNode; EntryCount = 0; @@ -980,11 +980,11 @@ ui64 TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::ArcSaveAndDestroy(co template <class T, class D, class S> typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode::TArcSet::iterator TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode::TArcSet::Find(char ch) { - using namespace NCompTriePrivate; - iterator it = LowerBound(this->begin(), this->end(), ch, TCmp()); - + using namespace NCompTriePrivate; + iterator it = LowerBound(this->begin(), this->end(), ch, TCmp()); + if (it != this->end() && it->Label[0] == (unsigned char)ch) { - return it; + return it; } return this->end(); @@ -1005,7 +1005,7 @@ typename TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode::TArcSet:: template <class T, class D, class S> void TCompactTrieBuilder<T, D, S>::TCompactTrieBuilderImpl::TNode::TArcSet::Add(const TBlob& s, TNode* node) { - using namespace NCompTriePrivate; + using namespace NCompTriePrivate; this->insert(LowerBound(this->begin(), this->end(), s[0], TCmp()), TArc(s, node)); } diff --git a/library/cpp/containers/comptrie/comptrie_impl.cpp b/library/cpp/containers/comptrie/comptrie_impl.cpp index a116ab6d1e..a293b45d1c 100644 --- a/library/cpp/containers/comptrie/comptrie_impl.cpp +++ b/library/cpp/containers/comptrie/comptrie_impl.cpp @@ -6,34 +6,34 @@ // Unpack the leaf value. The algorithm can store up to 8 full bytes in leafs. namespace NCompactTrie { - size_t MeasureOffset(size_t offset) { - int n = 0; - - while (offset) { - offset >>= 8; - ++n; - } - - return n; + size_t MeasureOffset(size_t offset) { + int n = 0; + + while (offset) { + offset >>= 8; + ++n; + } + + return n; } - - size_t PackOffset(char* buffer, size_t offset) { - size_t len = MeasureOffset(offset); - size_t i = len; - - while (i--) { - buffer[i] = (char)(offset & 0xFF); - offset >>= 8; - } - - return len; + + size_t PackOffset(char* buffer, size_t offset) { + size_t len = MeasureOffset(offset); + size_t i = len; + + while (i--) { + buffer[i] = (char)(offset & 0xFF); + offset >>= 8; + } + + return len; } - - void ShowProgress(size_t n) { - if (n % 1000000 == 0) + + void ShowProgress(size_t n) { + if (n % 1000000 == 0) Cerr << n << ", RSS=" << (TRusage::Get().MaxRss >> 20) << "mb" << Endl; - else if (n % 20000 == 0) - Cerr << "."; - } + else if (n % 20000 == 0) + Cerr << "."; + } } diff --git a/library/cpp/containers/comptrie/comptrie_impl.h b/library/cpp/containers/comptrie/comptrie_impl.h index f41c38311a..f4b0bf072e 100644 --- a/library/cpp/containers/comptrie/comptrie_impl.h +++ b/library/cpp/containers/comptrie/comptrie_impl.h @@ -9,10 +9,10 @@ // NCompactTrie namespace NCompactTrie { - const char MT_FINAL = '\x80'; - const char MT_NEXT = '\x40'; + const char MT_FINAL = '\x80'; + const char MT_NEXT = '\x40'; const char MT_SIZEMASK = '\x07'; - const size_t MT_LEFTSHIFT = 3; + const size_t MT_LEFTSHIFT = 3; const size_t MT_RIGHTSHIFT = 0; Y_FORCE_INLINE size_t UnpackOffset(const char* p, size_t len); @@ -72,22 +72,22 @@ namespace NCompTriePrivate { typedef TUtf32String TResult; }; -} +} -namespace NCompTriePrivate { - struct TCmp { - template <class T> - inline bool operator()(const T& l, const T& r) { +namespace NCompTriePrivate { + struct TCmp { + template <class T> + inline bool operator()(const T& l, const T& r) { return (unsigned char)(l.Label[0]) < (unsigned char)(r.Label[0]); - } - - template <class T> - inline bool operator()(const T& l, char r) { + } + + template <class T> + inline bool operator()(const T& l, char r) { return (unsigned char)(l.Label[0]) < (unsigned char)r; - } - }; -} - + } + }; +} + namespace NCompactTrie { static inline ui64 ArcSaveOffset(size_t offset, IOutputStream& os) { using namespace NCompactTrie; @@ -184,7 +184,7 @@ namespace NCompactTrie { // Returns true if the symbol was succesfully found in the trie, false otherwise. template <typename TSymbol, class TPacker> Y_FORCE_INLINE bool Advance(const char*& datapos, const char* const dataend, const char*& value, - TSymbol label, TPacker packer) { + TSymbol label, TPacker packer) { Y_ASSERT(datapos < dataend); char flags = MT_NEXT; for (int i = (int)ExtraBits<TSymbol>(); i >= 0; i -= 8) { diff --git a/library/cpp/containers/comptrie/comptrie_trie.h b/library/cpp/containers/comptrie/comptrie_trie.h index 40ec1e52b3..1fa8794897 100644 --- a/library/cpp/containers/comptrie/comptrie_trie.h +++ b/library/cpp/containers/comptrie/comptrie_trie.h @@ -29,7 +29,7 @@ template <class TTrie> class TPrefixIterator; // in case of <char> specialization cannot distinguish between "" and "\0" keys -template <class T = char, class D = ui64, class S = TCompactTriePacker<D>> +template <class T = char, class D = ui64, class S = TCompactTriePacker<D>> class TCompactTrie { public: typedef T TSymbol; @@ -56,12 +56,12 @@ public: TCompactTrie(const char* d, size_t len, TPacker packer); TCompactTrie(const char* d, size_t len) - : TCompactTrie{d, len, TPacker{}} { + : TCompactTrie{d, len, TPacker{}} { } TCompactTrie(const TBlob& data, TPacker packer); explicit TCompactTrie(const TBlob& data) - : TCompactTrie{data, TPacker{}} { + : TCompactTrie{data, TPacker{}} { } // Skipper should be initialized with &Packer, not with &other.Packer, so you have to redefine these. @@ -126,7 +126,7 @@ public: bool FindLongestPrefix(const TKeyBuf& key, size_t* prefixLen, TData* value = nullptr, bool* hasNext = nullptr) const { return FindLongestPrefix(key.data(), key.size(), prefixLen, value, hasNext); } - + // Return trie, containing all tails for the given key inline TCompactTrie<T, D, S> FindTails(const TSymbol* key, size_t keylen) const; TCompactTrie<T, D, S> FindTails(const TKeyBuf& key) const { @@ -146,17 +146,17 @@ public: typedef NCompactTrie::TOpaqueTrieIterator TOpaqueTrieIterator; typedef NCompactTrie::TOpaqueTrie TOpaqueTrie; friend class TCompactTrie; - TConstIterator(const TOpaqueTrie& trie, const char* emptyValue, bool atend, TPacker packer); // only usable from Begin() and End() methods + TConstIterator(const TOpaqueTrie& trie, const char* emptyValue, bool atend, TPacker packer); // only usable from Begin() and End() methods TConstIterator(const TOpaqueTrie& trie, const char* emptyValue, const TKeyBuf& key, TPacker packer); // only usable from UpperBound() method public: TConstIterator() = default; - bool IsEmpty() const { - return !Impl; - } // Almost no other method can be called. + bool IsEmpty() const { + return !Impl; + } // Almost no other method can be called. - bool operator==(const TConstIterator& other) const; - bool operator!=(const TConstIterator& other) const; + bool operator==(const TConstIterator& other) const; + bool operator!=(const TConstIterator& other) const; TConstIterator& operator++(); TConstIterator operator++(int /*unused*/); TConstIterator& operator--(); @@ -205,8 +205,8 @@ protected: void LookupPhrases(const char* datapos, size_t len, const TSymbol* key, size_t keylen, TVector<TPhraseMatch>& matches, TSymbol separator) const; }; -template <class T = char, class D = ui64, class S = TCompactTriePacker<D>> -class TCompactTrieHolder: public TCompactTrie<T, D, S>, NNonCopyable::TNonCopyable { +template <class T = char, class D = ui64, class S = TCompactTriePacker<D>> +class TCompactTrieHolder: public TCompactTrie<T, D, S>, NNonCopyable::TNonCopyable { private: typedef TCompactTrie<T, D, S> TBase; TArrayHolder<char> Storage; @@ -239,35 +239,35 @@ TCompactTrie<T, D, S>::TCompactTrie(const char* d, size_t len, TPacker packer) template <class T, class D, class S> TCompactTrie<T, D, S>::TCompactTrie(const char* emptyValue) : EmptyValue(emptyValue) -{ -} +{ +} template <class T, class D, class S> TCompactTrie<T, D, S>::TCompactTrie(const TBlob& data, const char* emptyValue, TPacker packer) : DataHolder(data) , EmptyValue(emptyValue) , Packer(packer) -{ -} +{ +} template <class T, class D, class S> TCompactTrie<T, D, S>::TCompactTrie(const TCompactTrie& other) - : DataHolder(other.DataHolder) - , EmptyValue(other.EmptyValue) - , Packer(other.Packer) -{ -} + : DataHolder(other.DataHolder) + , EmptyValue(other.EmptyValue) + , Packer(other.Packer) +{ +} template <class T, class D, class S> TCompactTrie<T, D, S>::TCompactTrie(TCompactTrie&& other) noexcept - : DataHolder(std::move(other.DataHolder)) - , EmptyValue(std::move(other.EmptyValue)) - , Packer(std::move(other.Packer)) -{ -} + : DataHolder(std::move(other.DataHolder)) + , EmptyValue(std::move(other.EmptyValue)) + , Packer(std::move(other.Packer)) +{ +} template <class T, class D, class S> -TCompactTrie<T, D, S>& TCompactTrie<T, D, S>::operator=(const TCompactTrie& other) { +TCompactTrie<T, D, S>& TCompactTrie<T, D, S>::operator=(const TCompactTrie& other) { if (this != &other) { DataHolder = other.DataHolder; EmptyValue = other.EmptyValue; @@ -529,7 +529,7 @@ bool TCompactTrie<T, D, S>::LookupLongestPrefix(const TSymbol* key, size_t keyle template <class T, class D, class S> void TCompactTrie<T, D, S>::LookupPhrases( const char* datapos, size_t len, const TSymbol* key, size_t keylen, - TVector<TPhraseMatch>& matches, TSymbol separator) const { + TVector<TPhraseMatch>& matches, TSymbol separator) const { using namespace NCompactTrie; matches.clear(); @@ -572,10 +572,10 @@ TCompactTrieHolder<T, D, S>::TCompactTrieHolder(IInputStream& is, size_t len) template <class T, class D, class S> TCompactTrie<T, D, S>::TConstIterator::TConstIterator(const TOpaqueTrie& trie, const char* emptyValue, bool atend, TPacker packer) - : Packer(packer) - , Impl(new TOpaqueTrieIterator(trie, emptyValue, atend)) -{ -} + : Packer(packer) + , Impl(new TOpaqueTrieIterator(trie, emptyValue, atend)) +{ +} template <class T, class D, class S> TCompactTrie<T, D, S>::TConstIterator::TConstIterator(const TOpaqueTrie& trie, const char* emptyValue, const TKeyBuf& key, TPacker packer) @@ -586,7 +586,7 @@ TCompactTrie<T, D, S>::TConstIterator::TConstIterator(const TOpaqueTrie& trie, c } template <class T, class D, class S> -bool TCompactTrie<T, D, S>::TConstIterator::operator==(const TConstIterator& other) const { +bool TCompactTrie<T, D, S>::TConstIterator::operator==(const TConstIterator& other) const { if (!Impl) return !other.Impl; if (!other.Impl) @@ -595,8 +595,8 @@ bool TCompactTrie<T, D, S>::TConstIterator::operator==(const TConstIterator& oth } template <class T, class D, class S> -bool TCompactTrie<T, D, S>::TConstIterator::operator!=(const TConstIterator& other) const { - return !operator==(other); +bool TCompactTrie<T, D, S>::TConstIterator::operator!=(const TConstIterator& other) const { + return !operator==(other); } template <class T, class D, class S> diff --git a/library/cpp/containers/comptrie/comptrie_ut.cpp b/library/cpp/containers/comptrie/comptrie_ut.cpp index 74bee09b5d..b4f84b3e4b 100644 --- a/library/cpp/containers/comptrie/comptrie_ut.cpp +++ b/library/cpp/containers/comptrie/comptrie_ut.cpp @@ -1,12 +1,12 @@ -#include <util/random/shuffle.h> +#include <util/random/shuffle.h> #include <library/cpp/testing/unittest/registar.h> - + #include <util/stream/output.h> #include <utility> - -#include <util/charset/wide.h> + +#include <util/charset/wide.h> #include <util/generic/algorithm.h> -#include <util/generic/buffer.h> +#include <util/generic/buffer.h> #include <util/generic/map.h> #include <util/generic/vector.h> #include <util/generic/ptr.h> @@ -14,100 +14,100 @@ #include <util/folder/dirut.h> -#include <util/random/random.h> +#include <util/random/random.h> #include <util/random/fast.h> - + #include <util/string/hex.h> #include <util/string/cast.h> -#include "comptrie.h" +#include "comptrie.h" #include "set.h" #include "first_symbol_iterator.h" #include "search_iterator.h" #include "pattern_searcher.h" - + #include <array> #include <iterator> -class TCompactTrieTest: public TTestBase { +class TCompactTrieTest: public TTestBase { private: UNIT_TEST_SUITE(TCompactTrieTest); - UNIT_TEST(TestTrie8); - UNIT_TEST(TestTrie16); - UNIT_TEST(TestTrie32); - - UNIT_TEST(TestFastTrie8); - UNIT_TEST(TestFastTrie16); - UNIT_TEST(TestFastTrie32); - - UNIT_TEST(TestMinimizedTrie8); - UNIT_TEST(TestMinimizedTrie16); - UNIT_TEST(TestMinimizedTrie32); - - UNIT_TEST(TestFastMinimizedTrie8); - UNIT_TEST(TestFastMinimizedTrie16); - UNIT_TEST(TestFastMinimizedTrie32); - - UNIT_TEST(TestTrieIterator8); - UNIT_TEST(TestTrieIterator16); - UNIT_TEST(TestTrieIterator32); - - UNIT_TEST(TestMinimizedTrieIterator8); - UNIT_TEST(TestMinimizedTrieIterator16); - UNIT_TEST(TestMinimizedTrieIterator32); - - UNIT_TEST(TestPhraseSearch); - UNIT_TEST(TestAddGet); - UNIT_TEST(TestEmpty); - UNIT_TEST(TestUninitializedNonEmpty); - UNIT_TEST(TestRandom); - UNIT_TEST(TestFindTails); - UNIT_TEST(TestPrefixGrouped); - UNIT_TEST(CrashTestPrefixGrouped); + UNIT_TEST(TestTrie8); + UNIT_TEST(TestTrie16); + UNIT_TEST(TestTrie32); + + UNIT_TEST(TestFastTrie8); + UNIT_TEST(TestFastTrie16); + UNIT_TEST(TestFastTrie32); + + UNIT_TEST(TestMinimizedTrie8); + UNIT_TEST(TestMinimizedTrie16); + UNIT_TEST(TestMinimizedTrie32); + + UNIT_TEST(TestFastMinimizedTrie8); + UNIT_TEST(TestFastMinimizedTrie16); + UNIT_TEST(TestFastMinimizedTrie32); + + UNIT_TEST(TestTrieIterator8); + UNIT_TEST(TestTrieIterator16); + UNIT_TEST(TestTrieIterator32); + + UNIT_TEST(TestMinimizedTrieIterator8); + UNIT_TEST(TestMinimizedTrieIterator16); + UNIT_TEST(TestMinimizedTrieIterator32); + + UNIT_TEST(TestPhraseSearch); + UNIT_TEST(TestAddGet); + UNIT_TEST(TestEmpty); + UNIT_TEST(TestUninitializedNonEmpty); + UNIT_TEST(TestRandom); + UNIT_TEST(TestFindTails); + UNIT_TEST(TestPrefixGrouped); + UNIT_TEST(CrashTestPrefixGrouped); UNIT_TEST(TestMergeFromFile); UNIT_TEST(TestMergeFromBuffer); - UNIT_TEST(TestUnique); - UNIT_TEST(TestAddRetValue); - UNIT_TEST(TestClear); + UNIT_TEST(TestUnique); + UNIT_TEST(TestAddRetValue); + UNIT_TEST(TestClear); - UNIT_TEST(TestIterateEmptyKey); + UNIT_TEST(TestIterateEmptyKey); - UNIT_TEST(TestTrieSet); + UNIT_TEST(TestTrieSet); - UNIT_TEST(TestTrieForVectorInt64); - UNIT_TEST(TestTrieForListInt64); - UNIT_TEST(TestTrieForSetInt64); + UNIT_TEST(TestTrieForVectorInt64); + UNIT_TEST(TestTrieForListInt64); + UNIT_TEST(TestTrieForSetInt64); - UNIT_TEST(TestTrieForVectorStroka); - UNIT_TEST(TestTrieForListStroka); - UNIT_TEST(TestTrieForSetStroka); + UNIT_TEST(TestTrieForVectorStroka); + UNIT_TEST(TestTrieForListStroka); + UNIT_TEST(TestTrieForSetStroka); - UNIT_TEST(TestTrieForVectorWtroka); - UNIT_TEST(TestTrieForVectorFloat); - UNIT_TEST(TestTrieForVectorDouble); + UNIT_TEST(TestTrieForVectorWtroka); + UNIT_TEST(TestTrieForVectorFloat); + UNIT_TEST(TestTrieForVectorDouble); - UNIT_TEST(TestTrieForListVectorInt64); - UNIT_TEST(TestTrieForPairWtrokaVectorInt64); + UNIT_TEST(TestTrieForListVectorInt64); + UNIT_TEST(TestTrieForPairWtrokaVectorInt64); - UNIT_TEST(TestEmptyValueOutOfOrder); - UNIT_TEST(TestFindLongestPrefixWithEmptyValue); + UNIT_TEST(TestEmptyValueOutOfOrder); + UNIT_TEST(TestFindLongestPrefixWithEmptyValue); - UNIT_TEST(TestSearchIterChar); - UNIT_TEST(TestSearchIterWchar); + UNIT_TEST(TestSearchIterChar); + UNIT_TEST(TestSearchIterWchar); UNIT_TEST(TestSearchIterWchar32) - UNIT_TEST(TestCopyAndAssignment); + UNIT_TEST(TestCopyAndAssignment); - UNIT_TEST(TestFirstSymbolIterator8); - UNIT_TEST(TestFirstSymbolIterator16); - UNIT_TEST(TestFirstSymbolIterator32); + UNIT_TEST(TestFirstSymbolIterator8); + UNIT_TEST(TestFirstSymbolIterator16); + UNIT_TEST(TestFirstSymbolIterator32); UNIT_TEST(TestFirstSymbolIteratorChar32); - UNIT_TEST(TestArrayPacker); + UNIT_TEST(TestArrayPacker); - UNIT_TEST(TestBuilderFindLongestPrefix); - UNIT_TEST(TestBuilderFindLongestPrefixWithEmptyValue); + UNIT_TEST(TestBuilderFindLongestPrefix); + UNIT_TEST(TestBuilderFindLongestPrefixWithEmptyValue); UNIT_TEST(TestPatternSearcherEmpty); UNIT_TEST(TestPatternSearcherSimple); @@ -147,7 +147,7 @@ private: TVector<TContainer> GetSampleVectorData(size_t nValues); template <class TContainer> TVector<TContainer> GetSampleTextVectorData(size_t nValues); - template <class T> + template <class T> void CheckEquality(const T& value1, const T& value2) const; template <class TContainer> void TestTrieWithContainers(const TVector<TUtf16String>& keys, const TVector<TContainer>& sampleData, TString methodName); @@ -265,38 +265,38 @@ public: UNIT_TEST_SUITE_REGISTRATION(TCompactTrieTest); -const char* TCompactTrieTest::SampleData[] = { - "", - "a", "b", "c", "d", - "aa", "ab", "ac", "ad", - "aaa", "aab", "aac", "aad", - "aba", "abb", "abc", "abd", - "fba", "fbb", "fbc", "fbd", - "fbbaa", - "c\x85\xA4\xBF" // Just something outside ASCII. +const char* TCompactTrieTest::SampleData[] = { + "", + "a", "b", "c", "d", + "aa", "ab", "ac", "ad", + "aaa", "aab", "aac", "aad", + "aba", "abb", "abc", "abd", + "fba", "fbb", "fbc", "fbd", + "fbbaa", + "c\x85\xA4\xBF" // Just something outside ASCII. }; -template <class T> +template <class T> typename TCompactTrie<T>::TKey MakeWideKey(const char* str, size_t len) { typename TCompactTrie<T>::TKey buffer; for (size_t i = 0; i < len; i++) { unsigned int ch = (str[i] & 0xFF); - buffer.push_back((T)(ch | ch << 8 | ch << 16 | ch << 24)); + buffer.push_back((T)(ch | ch << 8 | ch << 16 | ch << 24)); } return buffer; } -template <class T> +template <class T> typename TCompactTrie<T>::TKey MakeWideKey(const TString& str) { return MakeWideKey<T>(str.c_str(), str.length()); } -template <class T> +template <class T> typename TCompactTrie<T>::TKey MakeWideKey(const TStringBuf& buf) { return MakeWideKey<T>(buf.data(), buf.length()); } -template <class T> +template <class T> void TCompactTrieTest::CreateTrie(IOutputStream& out, bool minimize, bool useFastLayout) { TCompactTrieBuilder<T> builder; @@ -305,18 +305,18 @@ void TCompactTrieTest::CreateTrie(IOutputStream& out, bool minimize, bool useFas builder.Add(MakeWideKey<T>(i, len), len * 2); } - + TBufferOutput tmp2; IOutputStream& currentOutput = useFastLayout ? tmp2 : out; if (minimize) { TBufferOutput buftmp; builder.Save(buftmp); - CompactTrieMinimize<TCompactTriePacker<ui64>>(currentOutput, buftmp.Buffer().Data(), buftmp.Buffer().Size(), false); + CompactTrieMinimize<TCompactTriePacker<ui64>>(currentOutput, buftmp.Buffer().Data(), buftmp.Buffer().Size(), false); } else { builder.Save(currentOutput); } if (useFastLayout) { - CompactTrieMakeFastLayout<TCompactTriePacker<T>>(out, tmp2.Buffer().Data(), tmp2.Buffer().Size(), false); + CompactTrieMakeFastLayout<TCompactTriePacker<T>>(out, tmp2.Buffer().Data(), tmp2.Buffer().Size(), false); } } @@ -337,7 +337,7 @@ static bool LexicographicStep(TString& s) { return true; } -template <class T> +template <class T> void TCompactTrieTest::CheckUpperBound(const char* data, size_t datalen) { TCompactTrie<T> trie(data, datalen); typedef typename TCompactTrie<T>::TKey TKey; @@ -357,7 +357,7 @@ void TCompactTrieTest::CheckUpperBound(const char* data, size_t datalen) { } while (LexicographicStep(key)); } -template <class T> +template <class T> void TCompactTrieTest::CheckData(const char* data, size_t datalen) { TCompactTrie<T> trie(data, datalen); @@ -413,13 +413,13 @@ void TCompactTrieTest::CheckData(const char* data, size_t datalen) { UNIT_ASSERT(trie.FindLongestPrefix(key, &prefixLen, &value)); UNIT_ASSERT_EQUAL(prefixLen, 3); UNIT_ASSERT_EQUAL(6, value); - + value = 12345678; UNIT_ASSERT(!trie.Find(key, &value)); - UNIT_ASSERT_EQUAL(12345678, value); //Failed Find() should not change value + UNIT_ASSERT_EQUAL(12345678, value); //Failed Find() should not change value } -template <class T> +template <class T> void TCompactTrieTest::CheckIterator(const char* data, size_t datalen) { typedef typename TCompactTrie<T>::TKey TKey; typedef typename TCompactTrie<T>::TValueType TValue; @@ -441,10 +441,10 @@ void TCompactTrieTest::CheckIterator(const char* data, size_t datalen) { received.insert(*it); items.push_back(*it); entry_count++; - it++; + it++; } TMap<TKey, ui64> received2; - for (std::pair<TKey, ui64> x : trie) { + for (std::pair<TKey, ui64> x : trie) { received2.insert(x); } UNIT_ASSERT(entry_count == stored.size()); @@ -485,7 +485,7 @@ void TCompactTrieTest::CheckIterator(const char* data, size_t datalen) { UNIT_ASSERT(revIt != trie.End()); } -template <class T> +template <class T> void TCompactTrieTest::TestTrie(bool minimize, bool useFastLayout) { TBufferOutput bufout; CreateTrie<T>(bufout, minimize, useFastLayout); @@ -493,75 +493,75 @@ void TCompactTrieTest::TestTrie(bool minimize, bool useFastLayout) { CheckUpperBound<T>(bufout.Buffer().Data(), bufout.Buffer().Size()); } -template <class T> +template <class T> void TCompactTrieTest::TestTrieIterator(bool minimize) { TBufferOutput bufout; CreateTrie<T>(bufout, minimize, false); CheckIterator<T>(bufout.Buffer().Data(), bufout.Buffer().Size()); } -void TCompactTrieTest::TestTrie8() { - TestTrie<char>(false, false); -} -void TCompactTrieTest::TestTrie16() { - TestTrie<wchar16>(false, false); -} -void TCompactTrieTest::TestTrie32() { - TestTrie<wchar32>(false, false); -} - -void TCompactTrieTest::TestFastTrie8() { - TestTrie<char>(false, true); -} -void TCompactTrieTest::TestFastTrie16() { - TestTrie<wchar16>(false, true); -} -void TCompactTrieTest::TestFastTrie32() { - TestTrie<wchar32>(false, true); -} - -void TCompactTrieTest::TestMinimizedTrie8() { - TestTrie<char>(true, false); -} -void TCompactTrieTest::TestMinimizedTrie16() { - TestTrie<wchar16>(true, false); -} -void TCompactTrieTest::TestMinimizedTrie32() { - TestTrie<wchar32>(true, false); -} - -void TCompactTrieTest::TestFastMinimizedTrie8() { - TestTrie<char>(true, true); -} -void TCompactTrieTest::TestFastMinimizedTrie16() { - TestTrie<wchar16>(true, true); -} -void TCompactTrieTest::TestFastMinimizedTrie32() { - TestTrie<wchar32>(true, true); -} - -void TCompactTrieTest::TestTrieIterator8() { - TestTrieIterator<char>(false); -} -void TCompactTrieTest::TestTrieIterator16() { - TestTrieIterator<wchar16>(false); -} -void TCompactTrieTest::TestTrieIterator32() { - TestTrieIterator<wchar32>(false); -} - -void TCompactTrieTest::TestMinimizedTrieIterator8() { - TestTrieIterator<char>(true); -} -void TCompactTrieTest::TestMinimizedTrieIterator16() { - TestTrieIterator<wchar16>(true); -} -void TCompactTrieTest::TestMinimizedTrieIterator32() { - TestTrieIterator<wchar32>(true); -} +void TCompactTrieTest::TestTrie8() { + TestTrie<char>(false, false); +} +void TCompactTrieTest::TestTrie16() { + TestTrie<wchar16>(false, false); +} +void TCompactTrieTest::TestTrie32() { + TestTrie<wchar32>(false, false); +} + +void TCompactTrieTest::TestFastTrie8() { + TestTrie<char>(false, true); +} +void TCompactTrieTest::TestFastTrie16() { + TestTrie<wchar16>(false, true); +} +void TCompactTrieTest::TestFastTrie32() { + TestTrie<wchar32>(false, true); +} + +void TCompactTrieTest::TestMinimizedTrie8() { + TestTrie<char>(true, false); +} +void TCompactTrieTest::TestMinimizedTrie16() { + TestTrie<wchar16>(true, false); +} +void TCompactTrieTest::TestMinimizedTrie32() { + TestTrie<wchar32>(true, false); +} + +void TCompactTrieTest::TestFastMinimizedTrie8() { + TestTrie<char>(true, true); +} +void TCompactTrieTest::TestFastMinimizedTrie16() { + TestTrie<wchar16>(true, true); +} +void TCompactTrieTest::TestFastMinimizedTrie32() { + TestTrie<wchar32>(true, true); +} + +void TCompactTrieTest::TestTrieIterator8() { + TestTrieIterator<char>(false); +} +void TCompactTrieTest::TestTrieIterator16() { + TestTrieIterator<wchar16>(false); +} +void TCompactTrieTest::TestTrieIterator32() { + TestTrieIterator<wchar32>(false); +} + +void TCompactTrieTest::TestMinimizedTrieIterator8() { + TestTrieIterator<char>(true); +} +void TCompactTrieTest::TestMinimizedTrieIterator16() { + TestTrieIterator<wchar16>(true); +} +void TCompactTrieTest::TestMinimizedTrieIterator32() { + TestTrieIterator<wchar32>(true); +} void TCompactTrieTest::TestPhraseSearch() { - static const char* phrases[] = {"ab", "ab cd", "ab cd ef"}; + static const char* phrases[] = {"ab", "ab cd", "ab cd ef"}; static const char* const goodphrase = "ab cd ef gh"; static const char* const badphrase = "cd ef gh ab"; TBufferOutput bufout; @@ -651,7 +651,7 @@ static char RandChar() { } static TString RandStr(const size_t max) { - size_t len = RandomNumber<size_t>() % max; + size_t len = RandomNumber<size_t>() % max; TString key; for (size_t j = 0; j < len; ++j) key += RandChar(); @@ -664,7 +664,7 @@ void TCompactTrieTest::TestRandom(const size_t n, const size_t maxKeySize) { TCompactTrieBuilder<char, typename T::TData, T> builder; typedef TMap<TString, typename T::TData> TKeys; TKeys keys; - + typename T::TData dummy; for (size_t i = 0; i < n; ++i) { const TString key = RandStr(maxKeySize); @@ -677,7 +677,7 @@ void TCompactTrieTest::TestRandom(const size_t n, const size_t maxKeySize) { UNIT_ASSERT(dummy == val); } } - + TBufferStream stream; size_t len = builder.Save(stream); TCompactTrie<char, typename T::TData, T> trie(stream.Buffer().Data(), len); @@ -721,12 +721,12 @@ void TCompactTrieTest::TestRandom(const size_t n, const size_t maxKeySize) { } void TCompactTrieTest::TestRandom() { - TestRandom<TIntPacker<ui64>, true>(1000, 1000); - TestRandom<TIntPacker<int>, true>(100, 100); - TestRandom<TDummyPacker<ui64>, true>(0, 0); - TestRandom<TDummyPacker<ui64>, true>(100, 3); - TestRandom<TDummyPacker<ui64>, true>(100, 100); - TestRandom<TStrokaPacker, true>(100, 100); + TestRandom<TIntPacker<ui64>, true>(1000, 1000); + TestRandom<TIntPacker<int>, true>(100, 100); + TestRandom<TDummyPacker<ui64>, true>(0, 0); + TestRandom<TDummyPacker<ui64>, true>(100, 3); + TestRandom<TDummyPacker<ui64>, true>(100, 100); + TestRandom<TStrokaPacker, true>(100, 100); } void TCompactTrieTest::TestFindTailsImpl(const TString& prefix) { @@ -779,14 +779,14 @@ void TCompactTrieTest::TestPrefixGrouped() { TBuffer b1b; TCompactTrieBuilder<char, ui32> b1(CTBF_PREFIX_GROUPED); const char* data[] = { - "Kazan", - "Moscow", - "Monino", - "Murmansk", - "Fryanovo", - "Fryazino", - "Fryazevo", - "Tumen", + "Kazan", + "Moscow", + "Monino", + "Murmansk", + "Fryanovo", + "Fryazino", + "Fryazevo", + "Tumen", }; for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { @@ -937,15 +937,15 @@ void TCompactTrieTest::TestUnique() { void TCompactTrieTest::TestUniqueImpl(bool isPrefixGrouped) { TCompactTrieBuilder<char, ui32> builder(CTBF_UNIQUE | (isPrefixGrouped ? CTBF_PREFIX_GROUPED : CTBF_NONE)); const char* data[] = { - "Kazan", - "Moscow", - "Monino", - "Murmansk", - "Fryanovo", - "Fryazino", - "Fryazevo", - "Fry", - "Tumen", + "Kazan", + "Moscow", + "Monino", + "Murmansk", + "Fryanovo", + "Fryazino", + "Fryazevo", + "Fry", + "Tumen", }; for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { UNIT_ASSERT_C(builder.Add(data[i], strlen(data[i]), i + 1), i); @@ -963,15 +963,15 @@ void TCompactTrieTest::TestUniqueImpl(bool isPrefixGrouped) { void TCompactTrieTest::TestAddRetValue() { TCompactTrieBuilder<char, ui32> builder; const char* data[] = { - "Kazan", - "Moscow", - "Monino", - "Murmansk", - "Fryanovo", - "Fryazino", - "Fryazevo", - "Fry", - "Tumen", + "Kazan", + "Moscow", + "Monino", + "Murmansk", + "Fryanovo", + "Fryazino", + "Fryazevo", + "Fry", + "Tumen", }; for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) { UNIT_ASSERT(builder.Add(data[i], strlen(data[i]), i + 1)); @@ -982,7 +982,7 @@ void TCompactTrieTest::TestAddRetValue() { } } -void TCompactTrieTest::TestClear() { +void TCompactTrieTest::TestClear() { TCompactTrieBuilder<char, ui32> builder; const char* data[] = { "Kazan", @@ -1008,14 +1008,14 @@ void TCompactTrieTest::TestFindTails() { TestFindTailsImpl("aa"); TestFindTailsImpl("bb"); TestFindTailsImpl("fb"); - TestFindTailsImpl("fbc"); - TestFindTailsImpl("fbbaa"); + TestFindTailsImpl("fbc"); + TestFindTailsImpl("fbbaa"); } template <class T> -class TCompactTrieTest::TDummyPacker: public TNullPacker<T> { +class TCompactTrieTest::TDummyPacker: public TNullPacker<T> { public: - static T Data(const TString&) { + static T Data(const TString&) { T data; TNullPacker<T>().UnpackLeaf(nullptr, data); return data; @@ -1024,21 +1024,21 @@ public: typedef T TData; }; -class TCompactTrieTest::TStrokaPacker: public TCompactTriePacker<TString> { +class TCompactTrieTest::TStrokaPacker: public TCompactTriePacker<TString> { public: typedef TString TData; - static TString Data(const TString& str) { + static TString Data(const TString& str) { return str; } }; template <class T> -class TCompactTrieTest::TIntPacker: public TCompactTriePacker<T> { +class TCompactTrieTest::TIntPacker: public TCompactTriePacker<T> { public: typedef T TData; - static TData Data(const TString&) { + static TData Data(const TString&) { return RandomNumber<std::make_unsigned_t<T>>(); } }; @@ -1099,7 +1099,7 @@ void TCompactTrieTest::TestTrieSet() { UNIT_ASSERT(!empty.Has("d")); UNIT_ASSERT(!empty.Has("d")); - UNIT_ASSERT(empty.Has("")); // contains only empty string + UNIT_ASSERT(empty.Has("")); // contains only empty string } // Tests for trie with vector (list, set) values @@ -1119,7 +1119,7 @@ TVector<TContainer> TCompactTrieTest::GetSampleVectorData(size_t nValues) { for (size_t i = 0; i < nValues; i++) { data.push_back(TContainer()); for (size_t j = 0; j < i; j++) - data[i].insert(data[i].end(), (typename TContainer::value_type)((j == 3) ? 0 : (1 << (j * 5)))); + data[i].insert(data[i].end(), (typename TContainer::value_type)((j == 3) ? 0 : (1 << (j * 5)))); } return data; } @@ -1135,16 +1135,16 @@ TVector<TContainer> TCompactTrieTest::GetSampleTextVectorData(size_t nValues) { return data; } -template <class T> +template <class T> void TCompactTrieTest::CheckEquality(const T& value1, const T& value2) const { UNIT_ASSERT_VALUES_EQUAL(value1, value2); } -template <> -void TCompactTrieTest::CheckEquality<TVector<i64>>(const TVector<i64>& value1, const TVector<i64>& value2) const { - UNIT_ASSERT_VALUES_EQUAL(value1.size(), value2.size()); - for (size_t i = 0; i < value1.size(); i++) - UNIT_ASSERT_VALUES_EQUAL(value1[i], value2[i]); +template <> +void TCompactTrieTest::CheckEquality<TVector<i64>>(const TVector<i64>& value1, const TVector<i64>& value2) const { + UNIT_ASSERT_VALUES_EQUAL(value1.size(), value2.size()); + for (size_t i = 0; i < value1.size(); i++) + UNIT_ASSERT_VALUES_EQUAL(value1[i], value2[i]); } template <class TContainer> @@ -1171,9 +1171,9 @@ void TCompactTrieTest::TestTrieWithContainers(const TVector<TUtf16String>& keys, unlink(fileName.data()); } -template <> -void TCompactTrieTest::TestTrieWithContainers<std::pair<TUtf16String, TVector<i64>>>(const TVector<TUtf16String>& keys, const TVector<std::pair<TUtf16String, TVector<i64>>>& sampleData, TString methodName) { - typedef std::pair<TUtf16String, TVector<i64>> TContainer; +template <> +void TCompactTrieTest::TestTrieWithContainers<std::pair<TUtf16String, TVector<i64>>>(const TVector<TUtf16String>& keys, const TVector<std::pair<TUtf16String, TVector<i64>>>& sampleData, TString methodName) { + typedef std::pair<TUtf16String, TVector<i64>> TContainer; TString fileName = GetSystemTempDir() + "/TCompactTrieTest-TestTrieWithContainers-" + methodName; TCompactTrieBuilder<wchar16, TContainer> b; @@ -1194,63 +1194,63 @@ void TCompactTrieTest::TestTrieWithContainers<std::pair<TUtf16String, TVector<i6 } void TCompactTrieTest::TestTrieForVectorInt64() { - TestTrieWithContainers<TVector<i64>>(GetSampleKeys(10), GetSampleVectorData<TVector<i64>>(10), "v-i64"); + TestTrieWithContainers<TVector<i64>>(GetSampleKeys(10), GetSampleVectorData<TVector<i64>>(10), "v-i64"); } void TCompactTrieTest::TestTrieForListInt64() { - TestTrieWithContainers<TList<i64>>(GetSampleKeys(10), GetSampleVectorData<TList<i64>>(10), "l-i64"); + TestTrieWithContainers<TList<i64>>(GetSampleKeys(10), GetSampleVectorData<TList<i64>>(10), "l-i64"); } void TCompactTrieTest::TestTrieForSetInt64() { - TestTrieWithContainers<TSet<i64>>(GetSampleKeys(10), GetSampleVectorData<TSet<i64>>(10), "s-i64"); + TestTrieWithContainers<TSet<i64>>(GetSampleKeys(10), GetSampleVectorData<TSet<i64>>(10), "s-i64"); } void TCompactTrieTest::TestTrieForVectorStroka() { - TestTrieWithContainers<TVector<TString>>(GetSampleKeys(10), GetSampleTextVectorData<TVector<TString>>(10), "v-str"); + TestTrieWithContainers<TVector<TString>>(GetSampleKeys(10), GetSampleTextVectorData<TVector<TString>>(10), "v-str"); } void TCompactTrieTest::TestTrieForListStroka() { - TestTrieWithContainers<TList<TString>>(GetSampleKeys(10), GetSampleTextVectorData<TList<TString>>(10), "l-str"); + TestTrieWithContainers<TList<TString>>(GetSampleKeys(10), GetSampleTextVectorData<TList<TString>>(10), "l-str"); } void TCompactTrieTest::TestTrieForSetStroka() { - TestTrieWithContainers<TSet<TString>>(GetSampleKeys(10), GetSampleTextVectorData<TSet<TString>>(10), "s-str"); + TestTrieWithContainers<TSet<TString>>(GetSampleKeys(10), GetSampleTextVectorData<TSet<TString>>(10), "s-str"); } void TCompactTrieTest::TestTrieForVectorWtroka() { - TVector<TVector<TString>> data = GetSampleTextVectorData<TVector<TString>>(10); - TVector<TVector<TUtf16String>> wData; + TVector<TVector<TString>> data = GetSampleTextVectorData<TVector<TString>>(10); + TVector<TVector<TUtf16String>> wData; for (size_t i = 0; i < data.size(); i++) { wData.push_back(TVector<TUtf16String>()); for (size_t j = 0; j < data[i].size(); j++) - wData[i].push_back(UTF8ToWide(data[i][j])); + wData[i].push_back(UTF8ToWide(data[i][j])); } - TestTrieWithContainers<TVector<TUtf16String>>(GetSampleKeys(10), wData, "v-wtr"); + TestTrieWithContainers<TVector<TUtf16String>>(GetSampleKeys(10), wData, "v-wtr"); } void TCompactTrieTest::TestTrieForVectorFloat() { - TestTrieWithContainers<TVector<float>>(GetSampleKeys(10), GetSampleVectorData<TVector<float>>(10), "v-float"); + TestTrieWithContainers<TVector<float>>(GetSampleKeys(10), GetSampleVectorData<TVector<float>>(10), "v-float"); } void TCompactTrieTest::TestTrieForVectorDouble() { - TestTrieWithContainers<TVector<double>>(GetSampleKeys(10), GetSampleVectorData<TVector<double>>(10), "v-double"); + TestTrieWithContainers<TVector<double>>(GetSampleKeys(10), GetSampleVectorData<TVector<double>>(10), "v-double"); } void TCompactTrieTest::TestTrieForListVectorInt64() { TVector<i64> tmp; - tmp.push_back(0); - TList<TVector<i64>> dataElement(5, tmp); - TVector<TList<TVector<i64>>> data(10, dataElement); - TestTrieWithContainers<TList<TVector<i64>>>(GetSampleKeys(10), data, "l-v-i64"); + tmp.push_back(0); + TList<TVector<i64>> dataElement(5, tmp); + TVector<TList<TVector<i64>>> data(10, dataElement); + TestTrieWithContainers<TList<TVector<i64>>>(GetSampleKeys(10), data, "l-v-i64"); } void TCompactTrieTest::TestTrieForPairWtrokaVectorInt64() { TVector<TUtf16String> keys = GetSampleKeys(10); - TVector<TVector<i64>> values = GetSampleVectorData<TVector<i64>>(10); - TVector<std::pair<TUtf16String, TVector<i64>>> data; + TVector<TVector<i64>> values = GetSampleVectorData<TVector<i64>>(10); + TVector<std::pair<TUtf16String, TVector<i64>>> data; for (size_t i = 0; i < 10; i++) data.push_back(std::pair<TUtf16String, TVector<i64>>(keys[i] + u"_v", values[i])); - TestTrieWithContainers<std::pair<TUtf16String, TVector<i64>>>(keys, data, "pair-str-v-i64"); + TestTrieWithContainers<std::pair<TUtf16String, TVector<i64>>>(keys, data, "pair-str-v-i64"); } void TCompactTrieTest::TestEmptyValueOutOfOrder() { @@ -1468,7 +1468,7 @@ void TCompactTrieTest::TestArrayPacker() { void TCompactTrieTest::TestBuilderFindLongestPrefix() { const size_t sizes[] = {10, 100}; - const double branchProbabilities[] = {0.01, 0.1, 0.5, 0.9, 0.99}; + const double branchProbabilities[] = {0.01, 0.1, 0.5, 0.9, 0.99}; for (size_t size : sizes) { for (double branchProbability : branchProbabilities) { TestBuilderFindLongestPrefix(size, branchProbability, false, false); @@ -1498,7 +1498,7 @@ void TCompactTrieTest::TestBuilderFindLongestPrefix(size_t keysCount, double bra if (isPrefixGrouped) Sort(keys.begin(), keys.end()); else - Shuffle(keys.begin(), keys.end()); + Shuffle(keys.begin(), keys.end()); TCompactTrieBuilder<char, TString> builder(isPrefixGrouped ? CTBF_PREFIX_GROUPED : CTBF_NONE); const TString EMPTY_VALUE = "empty"; diff --git a/library/cpp/containers/comptrie/first_symbol_iterator.h b/library/cpp/containers/comptrie/first_symbol_iterator.h index d06135f06f..d3c521e7e6 100644 --- a/library/cpp/containers/comptrie/first_symbol_iterator.h +++ b/library/cpp/containers/comptrie/first_symbol_iterator.h @@ -6,7 +6,7 @@ namespace NCompactTrie { // Iterates over possible first symbols in a trie. // Allows one to get the symbol and the subtrie starting from it. - template <class TTrie> + template <class TTrie> class TFirstSymbolIterator { public: using TSymbol = typename TTrie::TSymbol; @@ -58,4 +58,4 @@ namespace NCompactTrie { TCopyPtr<TOpaqueTrieIterator> Impl; }; -} +} diff --git a/library/cpp/containers/comptrie/key_selector.h b/library/cpp/containers/comptrie/key_selector.h index 60466cef71..80191df744 100644 --- a/library/cpp/containers/comptrie/key_selector.h +++ b/library/cpp/containers/comptrie/key_selector.h @@ -4,24 +4,24 @@ #include <util/generic/string.h> #include <util/generic/strbuf.h> -template <class T> +template <class T> struct TCompactTrieKeySelector { typedef TVector<T> TKey; typedef TVector<T> TKeyBuf; }; -template <class TChar> +template <class TChar> struct TCompactTrieCharKeySelector { typedef TBasicString<TChar> TKey; typedef TBasicStringBuf<TChar> TKeyBuf; }; -template <> -struct TCompactTrieKeySelector<char>: public TCompactTrieCharKeySelector<char> { +template <> +struct TCompactTrieKeySelector<char>: public TCompactTrieCharKeySelector<char> { }; -template <> -struct TCompactTrieKeySelector<wchar16>: public TCompactTrieCharKeySelector<wchar16> { +template <> +struct TCompactTrieKeySelector<wchar16>: public TCompactTrieCharKeySelector<wchar16> { }; template <> diff --git a/library/cpp/containers/comptrie/leaf_skipper.h b/library/cpp/containers/comptrie/leaf_skipper.h index 3959258948..0bdfa11d63 100644 --- a/library/cpp/containers/comptrie/leaf_skipper.h +++ b/library/cpp/containers/comptrie/leaf_skipper.h @@ -9,11 +9,11 @@ namespace NCompactTrie { virtual ~ILeafSkipper() = default; }; - template <class TPacker> - class TPackerLeafSkipper: public ILeafSkipper { + template <class TPacker> + class TPackerLeafSkipper: public ILeafSkipper { private: const TPacker* Packer; - + public: TPackerLeafSkipper(const TPacker* packer) : Packer(packer) @@ -25,9 +25,9 @@ namespace NCompactTrie { } // For test purposes. - const TPacker* GetPacker() const { - return Packer; - } + const TPacker* GetPacker() const { + return Packer; + } }; // The data you need to traverse the trie without unpacking the values. @@ -40,13 +40,13 @@ namespace NCompactTrie { : Data(data) , Length(dataLength) , SkipFunction(skipFunction) - { - } + { + } bool operator==(const TOpaqueTrie& other) const { return Data == other.Data && - Length == other.Length && - &SkipFunction == &other.SkipFunction; + Length == other.Length && + &SkipFunction == &other.SkipFunction; } bool operator!=(const TOpaqueTrie& other) const { diff --git a/library/cpp/containers/comptrie/loader/loader.h b/library/cpp/containers/comptrie/loader/loader.h index ee10e9b451..d172efa541 100644 --- a/library/cpp/containers/comptrie/loader/loader.h +++ b/library/cpp/containers/comptrie/loader/loader.h @@ -9,7 +9,7 @@ template <class TrieType, size_t N> TrieType LoadTrieFromArchive(const TString& key, const unsigned char (&data)[N], - bool ignoreErrors = false) { + bool ignoreErrors = false) { TArchiveReader archive(TBlob::NoCopy(data, sizeof(data))); if (archive.Has(key)) { TAutoPtr<IInputStream> trie = archive.ObjectByKey(key); diff --git a/library/cpp/containers/comptrie/loader/loader_ut.cpp b/library/cpp/containers/comptrie/loader/loader_ut.cpp index 345063a31e..2e771357fe 100644 --- a/library/cpp/containers/comptrie/loader/loader_ut.cpp +++ b/library/cpp/containers/comptrie/loader/loader_ut.cpp @@ -6,25 +6,25 @@ using TDummyTrie = TCompactTrie<char, i32>; namespace { const unsigned char DATA[] = { -#include "data.inc" +#include "data.inc" }; -} +} Y_UNIT_TEST_SUITE(ArchiveLoaderTests) { Y_UNIT_TEST(BaseTest) { - TDummyTrie trie = LoadTrieFromArchive<TDummyTrie>("/dummy.trie", DATA, true); - UNIT_ASSERT_EQUAL(trie.Size(), 3); + TDummyTrie trie = LoadTrieFromArchive<TDummyTrie>("/dummy.trie", DATA, true); + UNIT_ASSERT_EQUAL(trie.Size(), 3); - const TString TrieKyes[3] = { - "zero", "one", "two"}; - i32 val = -1; - for (i32 i = 0; i < 3; ++i) { + const TString TrieKyes[3] = { + "zero", "one", "two"}; + i32 val = -1; + for (i32 i = 0; i < 3; ++i) { UNIT_ASSERT(trie.Find(TrieKyes[i].data(), TrieKyes[i].size(), &val)); - UNIT_ASSERT_EQUAL(i, val); - } + UNIT_ASSERT_EQUAL(i, val); + } - UNIT_CHECK_GENERATED_EXCEPTION( - LoadTrieFromArchive<TDummyTrie>("/noname.trie", DATA), - yexception); + UNIT_CHECK_GENERATED_EXCEPTION( + LoadTrieFromArchive<TDummyTrie>("/noname.trie", DATA), + yexception); } } diff --git a/library/cpp/containers/comptrie/make_fast_layout.cpp b/library/cpp/containers/comptrie/make_fast_layout.cpp index ade78d7899..693384ee7f 100644 --- a/library/cpp/containers/comptrie/make_fast_layout.cpp +++ b/library/cpp/containers/comptrie/make_fast_layout.cpp @@ -24,444 +24,444 @@ // (there is not much difference between these papers, actually). // namespace NCompactTrie { - static size_t FindSupportingPowerOf2(size_t n) { - size_t result = 1ull << (8 * sizeof(size_t) - 1); - while (result > n) { - result >>= 1; - } - return result; + static size_t FindSupportingPowerOf2(size_t n) { + size_t result = 1ull << (8 * sizeof(size_t) - 1); + while (result > n) { + result >>= 1; + } + return result; } - namespace { - class TTrieNodeSet { - public: + namespace { + class TTrieNodeSet { + public: TTrieNodeSet() = default; - explicit TTrieNodeSet(const TOpaqueTrie& trie) - : Body(trie.Length / (8 * MinNodeSize) + 1, 0) - { - } - - bool Has(size_t offset) const { - const size_t reducedOffset = ReducedOffset(offset); - return OffsetCell(reducedOffset) & OffsetMask(reducedOffset); - } - - void Add(size_t offset) { - const size_t reducedOffset = ReducedOffset(offset); - OffsetCell(reducedOffset) |= OffsetMask(reducedOffset); - } - - void Remove(size_t offset) { - const size_t reducedOffset = ReducedOffset(offset); - OffsetCell(reducedOffset) &= ~OffsetMask(reducedOffset); - } - - void Swap(TTrieNodeSet& other) { - Body.swap(other.Body); - } - - private: - static const size_t MinNodeSize = 2; - TVector<ui8> Body; - - static size_t ReducedOffset(size_t offset) { - return offset / MinNodeSize; - } - static ui8 OffsetMask(size_t reducedOffset) { - return 1 << (reducedOffset % 8); - } - ui8& OffsetCell(size_t reducedOffset) { - return Body.at(reducedOffset / 8); - } - const ui8& OffsetCell(size_t reducedOffset) const { - return Body.at(reducedOffset / 8); - } - }; - - //--------------------------------------------------------------------- - - class TTrieNodeCounts { - public: + explicit TTrieNodeSet(const TOpaqueTrie& trie) + : Body(trie.Length / (8 * MinNodeSize) + 1, 0) + { + } + + bool Has(size_t offset) const { + const size_t reducedOffset = ReducedOffset(offset); + return OffsetCell(reducedOffset) & OffsetMask(reducedOffset); + } + + void Add(size_t offset) { + const size_t reducedOffset = ReducedOffset(offset); + OffsetCell(reducedOffset) |= OffsetMask(reducedOffset); + } + + void Remove(size_t offset) { + const size_t reducedOffset = ReducedOffset(offset); + OffsetCell(reducedOffset) &= ~OffsetMask(reducedOffset); + } + + void Swap(TTrieNodeSet& other) { + Body.swap(other.Body); + } + + private: + static const size_t MinNodeSize = 2; + TVector<ui8> Body; + + static size_t ReducedOffset(size_t offset) { + return offset / MinNodeSize; + } + static ui8 OffsetMask(size_t reducedOffset) { + return 1 << (reducedOffset % 8); + } + ui8& OffsetCell(size_t reducedOffset) { + return Body.at(reducedOffset / 8); + } + const ui8& OffsetCell(size_t reducedOffset) const { + return Body.at(reducedOffset / 8); + } + }; + + //--------------------------------------------------------------------- + + class TTrieNodeCounts { + public: TTrieNodeCounts() = default; - explicit TTrieNodeCounts(const TOpaqueTrie& trie) - : Body(trie.Length / MinNodeSize, 0) - , IsTree(false) - { - } - - size_t Get(size_t offset) const { - return IsTree ? 1 : Body.at(offset / MinNodeSize); - } - - void Inc(size_t offset) { - if (IsTree) { - return; - } - ui8& count = Body.at(offset / MinNodeSize); - if (count != MaxCount) { - ++count; - } - } - - size_t Dec(size_t offset) { - if (IsTree) { - return 0; - } - ui8& count = Body.at(offset / MinNodeSize); - Y_ASSERT(count > 0); - if (count != MaxCount) { - --count; - } - return count; - } - - void Swap(TTrieNodeCounts& other) { - Body.swap(other.Body); - ::DoSwap(IsTree, other.IsTree); - } - - void SetTreeMode() { - IsTree = true; - Body = TVector<ui8>(); - } - - private: - static const size_t MinNodeSize = 2; - static const ui8 MaxCount = 255; - - TVector<ui8> Body; - bool IsTree = false; - }; - - //---------------------------------------------------------- - - class TOffsetIndex { - public: - // In all methods: - // Key --- offset from the beginning of the old trie. - // Value --- offset from the end of the new trie. - - explicit TOffsetIndex(TTrieNodeCounts& counts) { - ParentCounts.Swap(counts); - } - - void Add(size_t key, size_t value) { - Data[key] = value; - } - - size_t Size() const { - return Data.size(); - } - - size_t Get(size_t key) { - auto pos = Data.find(key); - if (pos == Data.end()) { - ythrow yexception() << "Bad node walking order: trying to get node offset too early or too many times!"; - } - size_t result = pos->second; - if (ParentCounts.Dec(key) == 0) { - // We don't need this offset any more. - Data.erase(pos); - } - return result; - } - - private: - TTrieNodeCounts ParentCounts; - THashMap<size_t, size_t> Data; - }; - - //--------------------------------------------------------------------------------------- - - class TTrieMeasurer { - public: - TTrieMeasurer(const TOpaqueTrie& trie, bool verbose); - void Measure(); - - size_t GetDepth() const { - return Depth; - } - - size_t GetNodeCount() const { - return NodeCount; - } - - size_t GetUnminimizedNodeCount() const { - return UnminimizedNodeCount; - } - - bool IsTree() const { - return NodeCount == UnminimizedNodeCount; - } - - TTrieNodeCounts& GetParentCounts() { - return ParentCounts; - } - - const TOpaqueTrie& GetTrie() const { - return Trie; - } - - private: - const TOpaqueTrie& Trie; - size_t Depth; - TTrieNodeCounts ParentCounts; - size_t NodeCount; - size_t UnminimizedNodeCount; - const bool Verbose; - - // returns depth, increments NodeCount. - size_t MeasureSubtrie(size_t rootOffset, bool isNewPath); - }; - - TTrieMeasurer::TTrieMeasurer(const TOpaqueTrie& trie, bool verbose) - : Trie(trie) - , Depth(0) - , ParentCounts(trie) - , NodeCount(0) - , UnminimizedNodeCount(0) - , Verbose(verbose) - { - Y_ASSERT(Trie.Data); + explicit TTrieNodeCounts(const TOpaqueTrie& trie) + : Body(trie.Length / MinNodeSize, 0) + , IsTree(false) + { + } + + size_t Get(size_t offset) const { + return IsTree ? 1 : Body.at(offset / MinNodeSize); + } + + void Inc(size_t offset) { + if (IsTree) { + return; + } + ui8& count = Body.at(offset / MinNodeSize); + if (count != MaxCount) { + ++count; + } + } + + size_t Dec(size_t offset) { + if (IsTree) { + return 0; + } + ui8& count = Body.at(offset / MinNodeSize); + Y_ASSERT(count > 0); + if (count != MaxCount) { + --count; + } + return count; + } + + void Swap(TTrieNodeCounts& other) { + Body.swap(other.Body); + ::DoSwap(IsTree, other.IsTree); + } + + void SetTreeMode() { + IsTree = true; + Body = TVector<ui8>(); + } + + private: + static const size_t MinNodeSize = 2; + static const ui8 MaxCount = 255; + + TVector<ui8> Body; + bool IsTree = false; + }; + + //---------------------------------------------------------- + + class TOffsetIndex { + public: + // In all methods: + // Key --- offset from the beginning of the old trie. + // Value --- offset from the end of the new trie. + + explicit TOffsetIndex(TTrieNodeCounts& counts) { + ParentCounts.Swap(counts); + } + + void Add(size_t key, size_t value) { + Data[key] = value; + } + + size_t Size() const { + return Data.size(); + } + + size_t Get(size_t key) { + auto pos = Data.find(key); + if (pos == Data.end()) { + ythrow yexception() << "Bad node walking order: trying to get node offset too early or too many times!"; + } + size_t result = pos->second; + if (ParentCounts.Dec(key) == 0) { + // We don't need this offset any more. + Data.erase(pos); + } + return result; + } + + private: + TTrieNodeCounts ParentCounts; + THashMap<size_t, size_t> Data; + }; + + //--------------------------------------------------------------------------------------- + + class TTrieMeasurer { + public: + TTrieMeasurer(const TOpaqueTrie& trie, bool verbose); + void Measure(); + + size_t GetDepth() const { + return Depth; + } + + size_t GetNodeCount() const { + return NodeCount; + } + + size_t GetUnminimizedNodeCount() const { + return UnminimizedNodeCount; + } + + bool IsTree() const { + return NodeCount == UnminimizedNodeCount; + } + + TTrieNodeCounts& GetParentCounts() { + return ParentCounts; + } + + const TOpaqueTrie& GetTrie() const { + return Trie; + } + + private: + const TOpaqueTrie& Trie; + size_t Depth; + TTrieNodeCounts ParentCounts; + size_t NodeCount; + size_t UnminimizedNodeCount; + const bool Verbose; + + // returns depth, increments NodeCount. + size_t MeasureSubtrie(size_t rootOffset, bool isNewPath); + }; + + TTrieMeasurer::TTrieMeasurer(const TOpaqueTrie& trie, bool verbose) + : Trie(trie) + , Depth(0) + , ParentCounts(trie) + , NodeCount(0) + , UnminimizedNodeCount(0) + , Verbose(verbose) + { + Y_ASSERT(Trie.Data); + } + + void TTrieMeasurer::Measure() { + if (Verbose) { + Cerr << "Measuring the trie..." << Endl; + } + NodeCount = 0; + UnminimizedNodeCount = 0; + Depth = MeasureSubtrie(0, true); + if (IsTree()) { + ParentCounts.SetTreeMode(); + } + if (Verbose) { + Cerr << "Unminimized node count: " << UnminimizedNodeCount << Endl; + Cerr << "Trie depth: " << Depth << Endl; + Cerr << "Node count: " << NodeCount << Endl; + } + } + + // A chain of nodes linked by forward links + // is considered one node with many left and right children + // for depth measuring here and in + // TVanEmdeBoasReverseNodeEnumerator::FindDescendants. + size_t TTrieMeasurer::MeasureSubtrie(size_t rootOffset, bool isNewPath) { + Y_ASSERT(rootOffset < Trie.Length); + TNode node(Trie.Data, rootOffset, Trie.SkipFunction); + size_t depth = 0; + for (;;) { + ++UnminimizedNodeCount; + if (Verbose) { + ShowProgress(UnminimizedNodeCount); + } + if (isNewPath) { + if (ParentCounts.Get(node.GetOffset()) > 0) { + isNewPath = false; + } else { + ++NodeCount; + } + ParentCounts.Inc(node.GetOffset()); + } + if (node.GetLeftOffset()) { + depth = Max(depth, 1 + MeasureSubtrie(node.GetLeftOffset(), isNewPath)); + } + if (node.GetRightOffset()) { + depth = Max(depth, 1 + MeasureSubtrie(node.GetRightOffset(), isNewPath)); + } + if (node.GetForwardOffset()) { + node = TNode(Trie.Data, node.GetForwardOffset(), Trie.SkipFunction); + } else { + break; + } + } + return depth; } - - void TTrieMeasurer::Measure() { - if (Verbose) { - Cerr << "Measuring the trie..." << Endl; - } - NodeCount = 0; - UnminimizedNodeCount = 0; - Depth = MeasureSubtrie(0, true); - if (IsTree()) { - ParentCounts.SetTreeMode(); - } - if (Verbose) { - Cerr << "Unminimized node count: " << UnminimizedNodeCount << Endl; - Cerr << "Trie depth: " << Depth << Endl; - Cerr << "Node count: " << NodeCount << Endl; - } - } - - // A chain of nodes linked by forward links - // is considered one node with many left and right children - // for depth measuring here and in - // TVanEmdeBoasReverseNodeEnumerator::FindDescendants. - size_t TTrieMeasurer::MeasureSubtrie(size_t rootOffset, bool isNewPath) { - Y_ASSERT(rootOffset < Trie.Length); - TNode node(Trie.Data, rootOffset, Trie.SkipFunction); - size_t depth = 0; - for (;;) { - ++UnminimizedNodeCount; - if (Verbose) { - ShowProgress(UnminimizedNodeCount); - } - if (isNewPath) { - if (ParentCounts.Get(node.GetOffset()) > 0) { - isNewPath = false; - } else { - ++NodeCount; - } - ParentCounts.Inc(node.GetOffset()); - } - if (node.GetLeftOffset()) { - depth = Max(depth, 1 + MeasureSubtrie(node.GetLeftOffset(), isNewPath)); - } - if (node.GetRightOffset()) { - depth = Max(depth, 1 + MeasureSubtrie(node.GetRightOffset(), isNewPath)); - } - if (node.GetForwardOffset()) { - node = TNode(Trie.Data, node.GetForwardOffset(), Trie.SkipFunction); - } else { - break; - } - } - return depth; - } - - //-------------------------------------------------------------------------------------- - - using TLevelNodes = TVector<size_t>; - - struct TLevel { - size_t Depth; - TLevelNodes Nodes; - - explicit TLevel(size_t depth) - : Depth(depth) - { - } - }; - - //---------------------------------------------------------------------------------------- - - class TVanEmdeBoasReverseNodeEnumerator: public TReverseNodeEnumerator { - public: - TVanEmdeBoasReverseNodeEnumerator(TTrieMeasurer& measurer, bool verbose) - : Fresh(true) - , Trie(measurer.GetTrie()) - , Depth(measurer.GetDepth()) - , MaxIndexSize(0) - , BackIndex(measurer.GetParentCounts()) - , ProcessedNodes(measurer.GetTrie()) - , Verbose(verbose) - { - } - - bool Move() override { - if (!Fresh) { - CentralWord.pop_back(); - } - if (CentralWord.empty()) { - return MoveCentralWordStart(); - } - return true; - } - - const TNode& Get() const { - return CentralWord.back(); - } - - size_t GetLeafLength() const override { - return Get().GetLeafLength(); - } - - // Returns recalculated offset from the end of the current node. - size_t PrepareOffset(size_t absoffset, size_t resultLength) { - if (!absoffset) - return NPOS; - return resultLength - BackIndex.Get(absoffset); - } - - size_t RecreateNode(char* buffer, size_t resultLength) override { - TWriteableNode newNode(Get(), Trie.Data); - newNode.ForwardOffset = PrepareOffset(Get().GetForwardOffset(), resultLength); - newNode.LeftOffset = PrepareOffset(Get().GetLeftOffset(), resultLength); - newNode.RightOffset = PrepareOffset(Get().GetRightOffset(), resultLength); - - const size_t len = newNode.Pack(buffer); - ProcessedNodes.Add(Get().GetOffset()); - BackIndex.Add(Get().GetOffset(), resultLength + len); - MaxIndexSize = Max(MaxIndexSize, BackIndex.Size()); - return len; - } - - private: - bool Fresh; - TOpaqueTrie Trie; - size_t Depth; - size_t MaxIndexSize; - - TVector<TLevel> Trace; - TOffsetIndex BackIndex; - TVector<TNode> CentralWord; - TTrieNodeSet ProcessedNodes; - - const bool Verbose; - - private: - bool IsVisited(size_t offset) const { - return ProcessedNodes.Has(offset); - } - - void AddCascade(size_t step) { - Y_ASSERT(!(step & (step - 1))); // Should be a power of 2. - while (step > 0) { - size_t root = Trace.back().Nodes.back(); - TLevel level(Trace.back().Depth + step); - Trace.push_back(level); - size_t actualStep = FindSupportingPowerOf2(FindDescendants(root, step, Trace.back().Nodes)); - if (actualStep != step) { - // Retry with a smaller step. - Y_ASSERT(actualStep < step); - step = actualStep; - Trace.pop_back(); - } else { - step /= 2; + + //-------------------------------------------------------------------------------------- + + using TLevelNodes = TVector<size_t>; + + struct TLevel { + size_t Depth; + TLevelNodes Nodes; + + explicit TLevel(size_t depth) + : Depth(depth) + { + } + }; + + //---------------------------------------------------------------------------------------- + + class TVanEmdeBoasReverseNodeEnumerator: public TReverseNodeEnumerator { + public: + TVanEmdeBoasReverseNodeEnumerator(TTrieMeasurer& measurer, bool verbose) + : Fresh(true) + , Trie(measurer.GetTrie()) + , Depth(measurer.GetDepth()) + , MaxIndexSize(0) + , BackIndex(measurer.GetParentCounts()) + , ProcessedNodes(measurer.GetTrie()) + , Verbose(verbose) + { + } + + bool Move() override { + if (!Fresh) { + CentralWord.pop_back(); + } + if (CentralWord.empty()) { + return MoveCentralWordStart(); + } + return true; + } + + const TNode& Get() const { + return CentralWord.back(); + } + + size_t GetLeafLength() const override { + return Get().GetLeafLength(); + } + + // Returns recalculated offset from the end of the current node. + size_t PrepareOffset(size_t absoffset, size_t resultLength) { + if (!absoffset) + return NPOS; + return resultLength - BackIndex.Get(absoffset); + } + + size_t RecreateNode(char* buffer, size_t resultLength) override { + TWriteableNode newNode(Get(), Trie.Data); + newNode.ForwardOffset = PrepareOffset(Get().GetForwardOffset(), resultLength); + newNode.LeftOffset = PrepareOffset(Get().GetLeftOffset(), resultLength); + newNode.RightOffset = PrepareOffset(Get().GetRightOffset(), resultLength); + + const size_t len = newNode.Pack(buffer); + ProcessedNodes.Add(Get().GetOffset()); + BackIndex.Add(Get().GetOffset(), resultLength + len); + MaxIndexSize = Max(MaxIndexSize, BackIndex.Size()); + return len; + } + + private: + bool Fresh; + TOpaqueTrie Trie; + size_t Depth; + size_t MaxIndexSize; + + TVector<TLevel> Trace; + TOffsetIndex BackIndex; + TVector<TNode> CentralWord; + TTrieNodeSet ProcessedNodes; + + const bool Verbose; + + private: + bool IsVisited(size_t offset) const { + return ProcessedNodes.Has(offset); + } + + void AddCascade(size_t step) { + Y_ASSERT(!(step & (step - 1))); // Should be a power of 2. + while (step > 0) { + size_t root = Trace.back().Nodes.back(); + TLevel level(Trace.back().Depth + step); + Trace.push_back(level); + size_t actualStep = FindSupportingPowerOf2(FindDescendants(root, step, Trace.back().Nodes)); + if (actualStep != step) { + // Retry with a smaller step. + Y_ASSERT(actualStep < step); + step = actualStep; + Trace.pop_back(); + } else { + step /= 2; + } + } + } + + void FillCentralWord() { + CentralWord.clear(); + CentralWord.push_back(TNode(Trie.Data, Trace.back().Nodes.back(), Trie.SkipFunction)); + // Do not check for epsilon links, as the traversal order now is different. + while (CentralWord.back().GetForwardOffset() && !IsVisited(CentralWord.back().GetForwardOffset())) { + CentralWord.push_back(TNode(Trie.Data, CentralWord.back().GetForwardOffset(), Trie.SkipFunction)); + } + } + + bool MoveCentralWordStart() { + do { + if (Fresh) { + TLevel root(0); + Trace.push_back(root); + Trace.back().Nodes.push_back(0); + const size_t sectionDepth = FindSupportingPowerOf2(Depth); + AddCascade(sectionDepth); + Fresh = false; + } else { + Trace.back().Nodes.pop_back(); + if (Trace.back().Nodes.empty() && Trace.size() == 1) { + if (Verbose) { + Cerr << "Max index size: " << MaxIndexSize << Endl; + Cerr << "Current index size: " << BackIndex.Size() << Endl; + } + // We just popped the root. + return false; + } + size_t lastStep = Trace.back().Depth - Trace[Trace.size() - 2].Depth; + if (Trace.back().Nodes.empty()) { + Trace.pop_back(); + } + AddCascade(lastStep / 2); + } + } while (IsVisited(Trace.back().Nodes.back())); + FillCentralWord(); + return true; + } + + // Returns the maximal depth it has reached while searching. + // This is a method because it needs OffsetIndex to skip processed nodes. + size_t FindDescendants(size_t rootOffset, size_t depth, TLevelNodes& result) const { + if (depth == 0) { + result.push_back(rootOffset); + return 0; + } + size_t actualDepth = 0; + TNode node(Trie.Data, rootOffset, Trie.SkipFunction); + for (;;) { + if (node.GetLeftOffset() && !IsVisited(node.GetLeftOffset())) { + actualDepth = Max(actualDepth, + 1 + FindDescendants(node.GetLeftOffset(), depth - 1, result)); } + if (node.GetRightOffset() && !IsVisited(node.GetRightOffset())) { + actualDepth = Max(actualDepth, + 1 + FindDescendants(node.GetRightOffset(), depth - 1, result)); + } + if (node.GetForwardOffset() && !IsVisited(node.GetForwardOffset())) { + node = TNode(Trie.Data, node.GetForwardOffset(), Trie.SkipFunction); + } else { + break; + } } + return actualDepth; } + }; - void FillCentralWord() { - CentralWord.clear(); - CentralWord.push_back(TNode(Trie.Data, Trace.back().Nodes.back(), Trie.SkipFunction)); - // Do not check for epsilon links, as the traversal order now is different. - while (CentralWord.back().GetForwardOffset() && !IsVisited(CentralWord.back().GetForwardOffset())) { - CentralWord.push_back(TNode(Trie.Data, CentralWord.back().GetForwardOffset(), Trie.SkipFunction)); - } - } - - bool MoveCentralWordStart() { - do { - if (Fresh) { - TLevel root(0); - Trace.push_back(root); - Trace.back().Nodes.push_back(0); - const size_t sectionDepth = FindSupportingPowerOf2(Depth); - AddCascade(sectionDepth); - Fresh = false; - } else { - Trace.back().Nodes.pop_back(); - if (Trace.back().Nodes.empty() && Trace.size() == 1) { - if (Verbose) { - Cerr << "Max index size: " << MaxIndexSize << Endl; - Cerr << "Current index size: " << BackIndex.Size() << Endl; - } - // We just popped the root. - return false; - } - size_t lastStep = Trace.back().Depth - Trace[Trace.size() - 2].Depth; - if (Trace.back().Nodes.empty()) { - Trace.pop_back(); - } - AddCascade(lastStep / 2); - } - } while (IsVisited(Trace.back().Nodes.back())); - FillCentralWord(); - return true; - } - - // Returns the maximal depth it has reached while searching. - // This is a method because it needs OffsetIndex to skip processed nodes. - size_t FindDescendants(size_t rootOffset, size_t depth, TLevelNodes& result) const { - if (depth == 0) { - result.push_back(rootOffset); - return 0; - } - size_t actualDepth = 0; - TNode node(Trie.Data, rootOffset, Trie.SkipFunction); - for (;;) { - if (node.GetLeftOffset() && !IsVisited(node.GetLeftOffset())) { - actualDepth = Max(actualDepth, - 1 + FindDescendants(node.GetLeftOffset(), depth - 1, result)); - } - if (node.GetRightOffset() && !IsVisited(node.GetRightOffset())) { - actualDepth = Max(actualDepth, - 1 + FindDescendants(node.GetRightOffset(), depth - 1, result)); - } - if (node.GetForwardOffset() && !IsVisited(node.GetForwardOffset())) { - node = TNode(Trie.Data, node.GetForwardOffset(), Trie.SkipFunction); - } else { - break; - } - } - return actualDepth; - } - }; - - } // Anonymous namespace. - - //----------------------------------------------------------------------------------- - - size_t RawCompactTrieFastLayoutImpl(IOutputStream& os, const TOpaqueTrie& trie, bool verbose) { - if (!trie.Data || !trie.Length) { + } // Anonymous namespace. + + //----------------------------------------------------------------------------------- + + size_t RawCompactTrieFastLayoutImpl(IOutputStream& os, const TOpaqueTrie& trie, bool verbose) { + if (!trie.Data || !trie.Length) { return 0; } - TTrieMeasurer mes(trie, verbose); - mes.Measure(); - TVanEmdeBoasReverseNodeEnumerator enumerator(mes, verbose); - return WriteTrieBackwards(os, enumerator, verbose); + TTrieMeasurer mes(trie, verbose); + mes.Measure(); + TVanEmdeBoasReverseNodeEnumerator enumerator(mes, verbose); + return WriteTrieBackwards(os, enumerator, verbose); } - + } diff --git a/library/cpp/containers/comptrie/make_fast_layout.h b/library/cpp/containers/comptrie/make_fast_layout.h index b8fab5d65b..cb488ffaa5 100644 --- a/library/cpp/containers/comptrie/make_fast_layout.h +++ b/library/cpp/containers/comptrie/make_fast_layout.h @@ -6,15 +6,15 @@ class IOutputStream; namespace NCompactTrie { - // Return value: size of the resulting trie. - size_t RawCompactTrieFastLayoutImpl(IOutputStream& os, const NCompactTrie::TOpaqueTrie& trie, bool verbose); - - // Return value: size of the resulting trie. - template <class TPacker> - size_t CompactTrieMakeFastLayoutImpl(IOutputStream& os, const char* data, size_t datalength, bool verbose, const TPacker* packer) { - TPackerLeafSkipper<TPacker> skipper(packer); - TOpaqueTrie trie(data, datalength, skipper); - return RawCompactTrieFastLayoutImpl(os, trie, verbose); - } + // Return value: size of the resulting trie. + size_t RawCompactTrieFastLayoutImpl(IOutputStream& os, const NCompactTrie::TOpaqueTrie& trie, bool verbose); + // Return value: size of the resulting trie. + template <class TPacker> + size_t CompactTrieMakeFastLayoutImpl(IOutputStream& os, const char* data, size_t datalength, bool verbose, const TPacker* packer) { + TPackerLeafSkipper<TPacker> skipper(packer); + TOpaqueTrie trie(data, datalength, skipper); + return RawCompactTrieFastLayoutImpl(os, trie, verbose); + } + } diff --git a/library/cpp/containers/comptrie/minimize.cpp b/library/cpp/containers/comptrie/minimize.cpp index 80d0b25217..f53511428f 100644 --- a/library/cpp/containers/comptrie/minimize.cpp +++ b/library/cpp/containers/comptrie/minimize.cpp @@ -8,352 +8,352 @@ #include <util/generic/algorithm.h> namespace NCompactTrie { - // Minimize the trie. The result is equivalent to the original - // trie, except that it takes less space (and has marginally lower - // performance, because of eventual epsilon links). - // The algorithm is as follows: starting from the largest pieces, we find - // nodes that have identical continuations (Daciuk's right language), - // and repack the trie. Repacking is done in-place, so memory is less - // of an issue; however, it may take considerable time. - - // IMPORTANT: never try to reminimize an already minimized trie or a trie with fast layout. - // Because of non-local structure and epsilon links, it won't work - // as you expect it to, and can destroy the trie in the making. - - namespace { - using TOffsetList = TVector<size_t>; - using TPieceIndex = THashMap<size_t, TOffsetList>; - - using TSizePair = std::pair<size_t, size_t>; - using TSizePairVector = TVector<TSizePair>; - using TSizePairVectorVector = TVector<TSizePairVector>; - - class TOffsetMap { - protected: - TSizePairVectorVector Data; - - public: - TOffsetMap() { - Data.reserve(0x10000); + // Minimize the trie. The result is equivalent to the original + // trie, except that it takes less space (and has marginally lower + // performance, because of eventual epsilon links). + // The algorithm is as follows: starting from the largest pieces, we find + // nodes that have identical continuations (Daciuk's right language), + // and repack the trie. Repacking is done in-place, so memory is less + // of an issue; however, it may take considerable time. + + // IMPORTANT: never try to reminimize an already minimized trie or a trie with fast layout. + // Because of non-local structure and epsilon links, it won't work + // as you expect it to, and can destroy the trie in the making. + + namespace { + using TOffsetList = TVector<size_t>; + using TPieceIndex = THashMap<size_t, TOffsetList>; + + using TSizePair = std::pair<size_t, size_t>; + using TSizePairVector = TVector<TSizePair>; + using TSizePairVectorVector = TVector<TSizePairVector>; + + class TOffsetMap { + protected: + TSizePairVectorVector Data; + + public: + TOffsetMap() { + Data.reserve(0x10000); + } + + void Add(size_t key, size_t value) { + size_t hikey = key & 0xFFFF; + + if (Data.size() <= hikey) + Data.resize(hikey + 1); + + TSizePairVector& sublist = Data[hikey]; + + for (auto& it : sublist) { + if (it.first == key) { + it.second = value; + + return; + } + } + + sublist.push_back(TSizePair(key, value)); + } + + bool Contains(size_t key) const { + return (Get(key) != 0); } - - void Add(size_t key, size_t value) { - size_t hikey = key & 0xFFFF; - - if (Data.size() <= hikey) - Data.resize(hikey + 1); - - TSizePairVector& sublist = Data[hikey]; - - for (auto& it : sublist) { - if (it.first == key) { - it.second = value; - - return; - } - } - - sublist.push_back(TSizePair(key, value)); - } - - bool Contains(size_t key) const { - return (Get(key) != 0); - } - - size_t Get(size_t key) const { - size_t hikey = key & 0xFFFF; - - if (Data.size() <= hikey) - return 0; - - const TSizePairVector& sublist = Data[hikey]; - - for (const auto& it : sublist) { - if (it.first == key) - return it.second; - } - - return 0; - } - }; - - class TOffsetDeltas { - protected: - TSizePairVector Data; - - public: - void Add(size_t key, size_t value) { - if (Data.empty()) { - if (key == value) - return; // no offset - } else { - TSizePair last = Data.back(); - - if (key <= last.first) { - Cerr << "Trouble: elements to offset delta list added in wrong order" << Endl; - - return; - } - - if (last.first + value == last.second + key) - return; // same offset - } - - Data.push_back(TSizePair(key, value)); - } - - size_t Get(size_t key) const { - if (Data.empty()) - return key; // difference is zero; - - if (key < Data.front().first) - return key; - - // Binary search for the highest entry in the list that does not exceed the key - size_t from = 0; - size_t to = Data.size() - 1; - - while (from < to) { - size_t midpoint = (from + to + 1) / 2; - - if (key < Data[midpoint].first) - to = midpoint - 1; - else - from = midpoint; - } - - TSizePair entry = Data[from]; - - return key - entry.first + entry.second; - } - }; - - class TPieceComparer { - private: - const char* Data; - const size_t Length; - - public: - TPieceComparer(const char* buf, size_t len) - : Data(buf) - , Length(len) - { + + size_t Get(size_t key) const { + size_t hikey = key & 0xFFFF; + + if (Data.size() <= hikey) + return 0; + + const TSizePairVector& sublist = Data[hikey]; + + for (const auto& it : sublist) { + if (it.first == key) + return it.second; + } + + return 0; + } + }; + + class TOffsetDeltas { + protected: + TSizePairVector Data; + + public: + void Add(size_t key, size_t value) { + if (Data.empty()) { + if (key == value) + return; // no offset + } else { + TSizePair last = Data.back(); + + if (key <= last.first) { + Cerr << "Trouble: elements to offset delta list added in wrong order" << Endl; + + return; + } + + if (last.first + value == last.second + key) + return; // same offset + } + + Data.push_back(TSizePair(key, value)); } - - bool operator()(size_t p1, const size_t p2) { - int res = memcmp(Data + p1, Data + p2, Length); - - if (res) - return (res > 0); - - return (p1 > p2); // the pieces are sorted in the reverse order of appearance - } - }; - - struct TBranchPoint { - TNode Node; - int Selector; - - public: - TBranchPoint() - : Selector(0) - { + + size_t Get(size_t key) const { + if (Data.empty()) + return key; // difference is zero; + + if (key < Data.front().first) + return key; + + // Binary search for the highest entry in the list that does not exceed the key + size_t from = 0; + size_t to = Data.size() - 1; + + while (from < to) { + size_t midpoint = (from + to + 1) / 2; + + if (key < Data[midpoint].first) + to = midpoint - 1; + else + from = midpoint; + } + + TSizePair entry = Data[from]; + + return key - entry.first + entry.second; + } + }; + + class TPieceComparer { + private: + const char* Data; + const size_t Length; + + public: + TPieceComparer(const char* buf, size_t len) + : Data(buf) + , Length(len) + { + } + + bool operator()(size_t p1, const size_t p2) { + int res = memcmp(Data + p1, Data + p2, Length); + + if (res) + return (res > 0); + + return (p1 > p2); // the pieces are sorted in the reverse order of appearance + } + }; + + struct TBranchPoint { + TNode Node; + int Selector; + + public: + TBranchPoint() + : Selector(0) + { + } + + TBranchPoint(const char* data, size_t offset, const ILeafSkipper& skipFunction) + : Node(data, offset, skipFunction) + , Selector(0) + { + } + + bool IsFinal() const { + return Node.IsFinal(); + } + + // NextNode returns child nodes, starting from the last node: Right, then Left, then Forward + size_t NextNode(const TOffsetMap& mergedNodes) { + while (Selector < 3) { + size_t nextOffset = 0; + + switch (++Selector) { + case 1: + if (Node.GetRightOffset()) + nextOffset = Node.GetRightOffset(); + break; + + case 2: + if (Node.GetLeftOffset()) + nextOffset = Node.GetLeftOffset(); + break; + + case 3: + if (Node.GetForwardOffset()) + nextOffset = Node.GetForwardOffset(); + break; + + default: + break; + } + + if (nextOffset && !mergedNodes.Contains(nextOffset)) + return nextOffset; + } + return 0; } - - TBranchPoint(const char* data, size_t offset, const ILeafSkipper& skipFunction) - : Node(data, offset, skipFunction) - , Selector(0) - { - } - - bool IsFinal() const { - return Node.IsFinal(); - } - - // NextNode returns child nodes, starting from the last node: Right, then Left, then Forward - size_t NextNode(const TOffsetMap& mergedNodes) { - while (Selector < 3) { - size_t nextOffset = 0; - - switch (++Selector) { - case 1: - if (Node.GetRightOffset()) - nextOffset = Node.GetRightOffset(); - break; - - case 2: - if (Node.GetLeftOffset()) - nextOffset = Node.GetLeftOffset(); - break; - - case 3: - if (Node.GetForwardOffset()) - nextOffset = Node.GetForwardOffset(); - break; - - default: - break; - } - - if (nextOffset && !mergedNodes.Contains(nextOffset)) - return nextOffset; - } - return 0; - } - }; - - class TMergingReverseNodeEnumerator: public TReverseNodeEnumerator { - private: - bool Fresh; - TOpaqueTrie Trie; - const TOffsetMap& MergeMap; - TVector<TBranchPoint> Trace; - TOffsetDeltas OffsetIndex; - - public: - TMergingReverseNodeEnumerator(const TOpaqueTrie& trie, const TOffsetMap& mergers) - : Fresh(true) - , Trie(trie) - , MergeMap(mergers) - { - } - - bool Move() override { - if (Fresh) { - Trace.push_back(TBranchPoint(Trie.Data, 0, Trie.SkipFunction)); - Fresh = false; - } else { - if (Trace.empty()) - return false; - - Trace.pop_back(); - - if (Trace.empty()) - return false; - } - - size_t nextnode = Trace.back().NextNode(MergeMap); - - while (nextnode) { - Trace.push_back(TBranchPoint(Trie.Data, nextnode, Trie.SkipFunction)); - nextnode = Trace.back().NextNode(MergeMap); - } - - return (!Trace.empty()); - } - - const TNode& Get() const { - return Trace.back().Node; - } - size_t GetLeafLength() const override { - return Get().GetLeafLength(); - } - - // Returns recalculated offset from the end of the current node - size_t PrepareOffset(size_t absoffset, size_t minilength) { - if (!absoffset) - return NPOS; - - if (MergeMap.Contains(absoffset)) - absoffset = MergeMap.Get(absoffset); - return minilength - OffsetIndex.Get(Trie.Length - absoffset); - } - - size_t RecreateNode(char* buffer, size_t resultLength) override { - TWriteableNode newNode(Get(), Trie.Data); - newNode.ForwardOffset = PrepareOffset(Get().GetForwardOffset(), resultLength); - newNode.LeftOffset = PrepareOffset(Get().GetLeftOffset(), resultLength); - newNode.RightOffset = PrepareOffset(Get().GetRightOffset(), resultLength); - - if (!buffer) - return newNode.Measure(); - - const size_t len = newNode.Pack(buffer); - OffsetIndex.Add(Trie.Length - Get().GetOffset(), resultLength + len); - - return len; - } - }; - + }; + + class TMergingReverseNodeEnumerator: public TReverseNodeEnumerator { + private: + bool Fresh; + TOpaqueTrie Trie; + const TOffsetMap& MergeMap; + TVector<TBranchPoint> Trace; + TOffsetDeltas OffsetIndex; + + public: + TMergingReverseNodeEnumerator(const TOpaqueTrie& trie, const TOffsetMap& mergers) + : Fresh(true) + , Trie(trie) + , MergeMap(mergers) + { + } + + bool Move() override { + if (Fresh) { + Trace.push_back(TBranchPoint(Trie.Data, 0, Trie.SkipFunction)); + Fresh = false; + } else { + if (Trace.empty()) + return false; + + Trace.pop_back(); + + if (Trace.empty()) + return false; + } + + size_t nextnode = Trace.back().NextNode(MergeMap); + + while (nextnode) { + Trace.push_back(TBranchPoint(Trie.Data, nextnode, Trie.SkipFunction)); + nextnode = Trace.back().NextNode(MergeMap); + } + + return (!Trace.empty()); + } + + const TNode& Get() const { + return Trace.back().Node; + } + size_t GetLeafLength() const override { + return Get().GetLeafLength(); + } + + // Returns recalculated offset from the end of the current node + size_t PrepareOffset(size_t absoffset, size_t minilength) { + if (!absoffset) + return NPOS; + + if (MergeMap.Contains(absoffset)) + absoffset = MergeMap.Get(absoffset); + return minilength - OffsetIndex.Get(Trie.Length - absoffset); + } + + size_t RecreateNode(char* buffer, size_t resultLength) override { + TWriteableNode newNode(Get(), Trie.Data); + newNode.ForwardOffset = PrepareOffset(Get().GetForwardOffset(), resultLength); + newNode.LeftOffset = PrepareOffset(Get().GetLeftOffset(), resultLength); + newNode.RightOffset = PrepareOffset(Get().GetRightOffset(), resultLength); + + if (!buffer) + return newNode.Measure(); + + const size_t len = newNode.Pack(buffer); + OffsetIndex.Add(Trie.Length - Get().GetOffset(), resultLength + len); + + return len; + } + }; + } - static void AddPiece(TPieceIndex& index, size_t offset, size_t len) { - index[len].push_back(offset); - } - - static TOffsetMap FindEquivalentSubtries(const TOpaqueTrie& trie, bool verbose, size_t minMergeSize) { - // Tree nodes, arranged by span length. - // When all nodes of a given size are considered, they pop off the queue. - TPieceIndex subtries; - TOffsetMap merger; - // Start walking the trie from head. - AddPiece(subtries, 0, trie.Length); - - size_t counter = 0; - // Now consider all nodes with sizeable continuations - for (size_t curlen = trie.Length; curlen >= minMergeSize && !subtries.empty(); curlen--) { - TPieceIndex::iterator iit = subtries.find(curlen); - - if (iit == subtries.end()) - continue; // fast forward to the next available length value - - TOffsetList& batch = iit->second; - TPieceComparer comparer(trie.Data, curlen); - Sort(batch.begin(), batch.end(), comparer); - - TOffsetList::iterator it = batch.begin(); - while (it != batch.end()) { - if (verbose) - ShowProgress(++counter); - - size_t offset = *it; - - // Fill the array with the subnodes of the element - TNode node(trie.Data, offset, trie.SkipFunction); - size_t end = offset + curlen; - if (size_t rightOffset = node.GetRightOffset()) { - AddPiece(subtries, rightOffset, end - rightOffset); - end = rightOffset; - } - if (size_t leftOffset = node.GetLeftOffset()) { - AddPiece(subtries, leftOffset, end - leftOffset); - end = leftOffset; - } - if (size_t forwardOffset = node.GetForwardOffset()) { - AddPiece(subtries, forwardOffset, end - forwardOffset); - } - - while (++it != batch.end()) { - // Find next different; until then, just add the offsets to the list of merged nodes. - size_t nextoffset = *it; - - if (memcmp(trie.Data + offset, trie.Data + nextoffset, curlen)) - break; - - merger.Add(nextoffset, offset); - } - } - - subtries.erase(curlen); - } - if (verbose) { - Cerr << counter << Endl; + static void AddPiece(TPieceIndex& index, size_t offset, size_t len) { + index[len].push_back(offset); + } + + static TOffsetMap FindEquivalentSubtries(const TOpaqueTrie& trie, bool verbose, size_t minMergeSize) { + // Tree nodes, arranged by span length. + // When all nodes of a given size are considered, they pop off the queue. + TPieceIndex subtries; + TOffsetMap merger; + // Start walking the trie from head. + AddPiece(subtries, 0, trie.Length); + + size_t counter = 0; + // Now consider all nodes with sizeable continuations + for (size_t curlen = trie.Length; curlen >= minMergeSize && !subtries.empty(); curlen--) { + TPieceIndex::iterator iit = subtries.find(curlen); + + if (iit == subtries.end()) + continue; // fast forward to the next available length value + + TOffsetList& batch = iit->second; + TPieceComparer comparer(trie.Data, curlen); + Sort(batch.begin(), batch.end(), comparer); + + TOffsetList::iterator it = batch.begin(); + while (it != batch.end()) { + if (verbose) + ShowProgress(++counter); + + size_t offset = *it; + + // Fill the array with the subnodes of the element + TNode node(trie.Data, offset, trie.SkipFunction); + size_t end = offset + curlen; + if (size_t rightOffset = node.GetRightOffset()) { + AddPiece(subtries, rightOffset, end - rightOffset); + end = rightOffset; + } + if (size_t leftOffset = node.GetLeftOffset()) { + AddPiece(subtries, leftOffset, end - leftOffset); + end = leftOffset; + } + if (size_t forwardOffset = node.GetForwardOffset()) { + AddPiece(subtries, forwardOffset, end - forwardOffset); + } + + while (++it != batch.end()) { + // Find next different; until then, just add the offsets to the list of merged nodes. + size_t nextoffset = *it; + + if (memcmp(trie.Data + offset, trie.Data + nextoffset, curlen)) + break; + + merger.Add(nextoffset, offset); + } + } + + subtries.erase(curlen); } - return merger; + if (verbose) { + Cerr << counter << Endl; + } + return merger; } - - size_t RawCompactTrieMinimizeImpl(IOutputStream& os, TOpaqueTrie& trie, bool verbose, size_t minMergeSize, EMinimizeMode mode) { - if (!trie.Data || !trie.Length) { - return 0; - } - - TOffsetMap merger = FindEquivalentSubtries(trie, verbose, minMergeSize); - TMergingReverseNodeEnumerator enumerator(trie, merger); - - if (mode == MM_DEFAULT) - return WriteTrieBackwards(os, enumerator, verbose); - else - return WriteTrieBackwardsNoAlloc(os, enumerator, trie, mode); + + size_t RawCompactTrieMinimizeImpl(IOutputStream& os, TOpaqueTrie& trie, bool verbose, size_t minMergeSize, EMinimizeMode mode) { + if (!trie.Data || !trie.Length) { + return 0; + } + + TOffsetMap merger = FindEquivalentSubtries(trie, verbose, minMergeSize); + TMergingReverseNodeEnumerator enumerator(trie, merger); + + if (mode == MM_DEFAULT) + return WriteTrieBackwards(os, enumerator, verbose); + else + return WriteTrieBackwardsNoAlloc(os, enumerator, trie, mode); } } diff --git a/library/cpp/containers/comptrie/minimize.h b/library/cpp/containers/comptrie/minimize.h index baaa431d04..2464571437 100644 --- a/library/cpp/containers/comptrie/minimize.h +++ b/library/cpp/containers/comptrie/minimize.h @@ -6,24 +6,24 @@ class IOutputStream; namespace NCompactTrie { - size_t MeasureOffset(size_t offset); + size_t MeasureOffset(size_t offset); - enum EMinimizeMode { - MM_DEFAULT, // alollocate new memory for minimized tree - MM_NOALLOC, // minimize tree in the same buffer - MM_INPLACE // do not write tree to the stream, but move to the buffer beginning - }; + enum EMinimizeMode { + MM_DEFAULT, // alollocate new memory for minimized tree + MM_NOALLOC, // minimize tree in the same buffer + MM_INPLACE // do not write tree to the stream, but move to the buffer beginning + }; - // Return value: size of the minimized trie. - size_t RawCompactTrieMinimizeImpl(IOutputStream& os, TOpaqueTrie& trie, bool verbose, size_t minMergeSize, EMinimizeMode mode); + // Return value: size of the minimized trie. + size_t RawCompactTrieMinimizeImpl(IOutputStream& os, TOpaqueTrie& trie, bool verbose, size_t minMergeSize, EMinimizeMode mode); - // Return value: size of the minimized trie. - template <class TPacker> - size_t CompactTrieMinimizeImpl(IOutputStream& os, const char* data, size_t datalength, bool verbose, const TPacker* packer, EMinimizeMode mode) { - TPackerLeafSkipper<TPacker> skipper(packer); - size_t minmerge = MeasureOffset(datalength); - TOpaqueTrie trie(data, datalength, skipper); - return RawCompactTrieMinimizeImpl(os, trie, verbose, minmerge, mode); - } + // Return value: size of the minimized trie. + template <class TPacker> + size_t CompactTrieMinimizeImpl(IOutputStream& os, const char* data, size_t datalength, bool verbose, const TPacker* packer, EMinimizeMode mode) { + TPackerLeafSkipper<TPacker> skipper(packer); + size_t minmerge = MeasureOffset(datalength); + TOpaqueTrie trie(data, datalength, skipper); + return RawCompactTrieMinimizeImpl(os, trie, verbose, minmerge, mode); + } } diff --git a/library/cpp/containers/comptrie/node.cpp b/library/cpp/containers/comptrie/node.cpp index 5fd22f15ec..cd98e6b4d6 100644 --- a/library/cpp/containers/comptrie/node.cpp +++ b/library/cpp/containers/comptrie/node.cpp @@ -6,74 +6,74 @@ #include <util/generic/yexception.h> namespace NCompactTrie { - TNode::TNode() - : Offset(0) - , LeafLength(0) - , CoreLength(0) - , Label(0) - { - for (auto& offset : Offsets) { - offset = 0; - } + TNode::TNode() + : Offset(0) + , LeafLength(0) + , CoreLength(0) + , Label(0) + { + for (auto& offset : Offsets) { + offset = 0; + } } - // We believe that epsilon links are found only on the forward-position and that afer jumping an epsilon link you come to an ordinary node. + // We believe that epsilon links are found only on the forward-position and that afer jumping an epsilon link you come to an ordinary node. + + TNode::TNode(const char* data, size_t offset, const ILeafSkipper& skipFunction) + : Offset(offset) + , LeafLength(0) + , CoreLength(0) + , Label(0) + { + for (auto& anOffset : Offsets) { + anOffset = 0; + } + if (!data) + return; // empty constructor + + const char* datapos = data + offset; + char flags = *(datapos++); + Y_ASSERT(!IsEpsilonLink(flags)); + Label = *(datapos++); - TNode::TNode(const char* data, size_t offset, const ILeafSkipper& skipFunction) - : Offset(offset) - , LeafLength(0) - , CoreLength(0) - , Label(0) - { - for (auto& anOffset : Offsets) { - anOffset = 0; - } - if (!data) - return; // empty constructor + size_t leftsize = LeftOffsetLen(flags); + size_t& leftOffset = Offsets[D_LEFT]; + leftOffset = UnpackOffset(datapos, leftsize); + if (leftOffset) { + leftOffset += Offset; + } + datapos += leftsize; - const char* datapos = data + offset; - char flags = *(datapos++); - Y_ASSERT(!IsEpsilonLink(flags)); - Label = *(datapos++); + size_t rightsize = RightOffsetLen(flags); + size_t& rightOffset = Offsets[D_RIGHT]; + rightOffset = UnpackOffset(datapos, rightsize); + if (rightOffset) { + rightOffset += Offset; + } + datapos += rightsize; + + if (flags & MT_FINAL) { + Offsets[D_FINAL] = datapos - data; + LeafLength = skipFunction.SkipLeaf(datapos); + } - size_t leftsize = LeftOffsetLen(flags); - size_t& leftOffset = Offsets[D_LEFT]; - leftOffset = UnpackOffset(datapos, leftsize); - if (leftOffset) { - leftOffset += Offset; - } - datapos += leftsize; - - size_t rightsize = RightOffsetLen(flags); - size_t& rightOffset = Offsets[D_RIGHT]; - rightOffset = UnpackOffset(datapos, rightsize); - if (rightOffset) { - rightOffset += Offset; - } - datapos += rightsize; - - if (flags & MT_FINAL) { - Offsets[D_FINAL] = datapos - data; - LeafLength = skipFunction.SkipLeaf(datapos); - } - - CoreLength = 2 + leftsize + rightsize + LeafLength; - if (flags & MT_NEXT) { - size_t& forwardOffset = Offsets[D_NEXT]; - forwardOffset = Offset + CoreLength; - // There might be an epsilon link at the forward position. - // If so, set the ForwardOffset to the value that points to the link's end. - const char* forwardPos = data + forwardOffset; - const char forwardFlags = *forwardPos; - if (IsEpsilonLink(forwardFlags)) { - // Jump through the epsilon link. - size_t epsilonOffset = UnpackOffset(forwardPos + 1, forwardFlags & MT_SIZEMASK); - if (!epsilonOffset) { - ythrow yexception() << "Corrupted epsilon link"; - } - forwardOffset += epsilonOffset; + CoreLength = 2 + leftsize + rightsize + LeafLength; + if (flags & MT_NEXT) { + size_t& forwardOffset = Offsets[D_NEXT]; + forwardOffset = Offset + CoreLength; + // There might be an epsilon link at the forward position. + // If so, set the ForwardOffset to the value that points to the link's end. + const char* forwardPos = data + forwardOffset; + const char forwardFlags = *forwardPos; + if (IsEpsilonLink(forwardFlags)) { + // Jump through the epsilon link. + size_t epsilonOffset = UnpackOffset(forwardPos + 1, forwardFlags & MT_SIZEMASK); + if (!epsilonOffset) { + ythrow yexception() << "Corrupted epsilon link"; + } + forwardOffset += epsilonOffset; } } } - + } diff --git a/library/cpp/containers/comptrie/node.h b/library/cpp/containers/comptrie/node.h index d6f4317db0..9a936998b5 100644 --- a/library/cpp/containers/comptrie/node.h +++ b/library/cpp/containers/comptrie/node.h @@ -3,78 +3,78 @@ #include <cstddef> namespace NCompactTrie { - class ILeafSkipper; + class ILeafSkipper; - enum TDirection { - D_LEFT, - D_FINAL, - D_NEXT, - D_RIGHT, - D_MAX - }; + enum TDirection { + D_LEFT, + D_FINAL, + D_NEXT, + D_RIGHT, + D_MAX + }; + + inline TDirection& operator++(TDirection& direction) { + direction = static_cast<TDirection>(direction + 1); + return direction; + } - inline TDirection& operator++(TDirection& direction) { - direction = static_cast<TDirection>(direction + 1); - return direction; - } + inline TDirection& operator--(TDirection& direction) { + direction = static_cast<TDirection>(direction - 1); + return direction; + } - inline TDirection& operator--(TDirection& direction) { - direction = static_cast<TDirection>(direction - 1); - return direction; - } + class TNode { + public: + TNode(); + // Processes epsilon links and sets ForwardOffset to correct value. Assumes an epsilon link doesn't point to an epsilon link. + TNode(const char* data, size_t offset, const ILeafSkipper& skipFunction); - class TNode { - public: - TNode(); - // Processes epsilon links and sets ForwardOffset to correct value. Assumes an epsilon link doesn't point to an epsilon link. - TNode(const char* data, size_t offset, const ILeafSkipper& skipFunction); + size_t GetOffset() const { + return Offset; + } - size_t GetOffset() const { - return Offset; - } + size_t GetLeafOffset() const { + return Offsets[D_FINAL]; + } + size_t GetLeafLength() const { + return LeafLength; + } + size_t GetCoreLength() const { + return CoreLength; + } - size_t GetLeafOffset() const { - return Offsets[D_FINAL]; - } - size_t GetLeafLength() const { - return LeafLength; - } - size_t GetCoreLength() const { - return CoreLength; - } + size_t GetOffsetByDirection(TDirection direction) const { + return Offsets[direction]; + } - size_t GetOffsetByDirection(TDirection direction) const { - return Offsets[direction]; - } + size_t GetForwardOffset() const { + return Offsets[D_NEXT]; + } + size_t GetLeftOffset() const { + return Offsets[D_LEFT]; + } + size_t GetRightOffset() const { + return Offsets[D_RIGHT]; + } + char GetLabel() const { + return Label; + } - size_t GetForwardOffset() const { - return Offsets[D_NEXT]; - } - size_t GetLeftOffset() const { - return Offsets[D_LEFT]; - } - size_t GetRightOffset() const { - return Offsets[D_RIGHT]; - } - char GetLabel() const { - return Label; - } + bool IsFinal() const { + return GetLeafOffset() != 0; + } - bool IsFinal() const { - return GetLeafOffset() != 0; - } + bool HasEpsilonLinkForward() const { + return GetForwardOffset() > Offset + CoreLength; + } + + private: + size_t Offsets[D_MAX]; + size_t Offset; + size_t LeafLength; + size_t CoreLength; - bool HasEpsilonLinkForward() const { - return GetForwardOffset() > Offset + CoreLength; - } + char Label; + }; - private: - size_t Offsets[D_MAX]; - size_t Offset; - size_t LeafLength; - size_t CoreLength; - - char Label; - }; - -} +} diff --git a/library/cpp/containers/comptrie/opaque_trie_iterator.cpp b/library/cpp/containers/comptrie/opaque_trie_iterator.cpp index 5fd3914be6..d1916358aa 100644 --- a/library/cpp/containers/comptrie/opaque_trie_iterator.cpp +++ b/library/cpp/containers/comptrie/opaque_trie_iterator.cpp @@ -3,229 +3,229 @@ #include "node.h" namespace NCompactTrie { - TOpaqueTrieIterator::TOpaqueTrieIterator(const TOpaqueTrie& trie, const char* emptyValue, bool atend, - size_t maxKeyLength) - : Trie(trie) - , EmptyValue(emptyValue) - , AtEmptyValue(emptyValue && !atend) - , MaxKeyLength(maxKeyLength) - { - if (!AtEmptyValue && !atend) - Forward(); - } - - bool TOpaqueTrieIterator::operator==(const TOpaqueTrieIterator& rhs) const { - return (Trie == rhs.Trie && - Forks == rhs.Forks && - EmptyValue == rhs.EmptyValue && - AtEmptyValue == rhs.AtEmptyValue && - MaxKeyLength == rhs.MaxKeyLength); - } - - bool TOpaqueTrieIterator::HasMaxKeyLength() const { - return MaxKeyLength != size_t(-1) && MeasureNarrowKey() == MaxKeyLength; - } - - bool TOpaqueTrieIterator::Forward() { - if (AtEmptyValue) { - AtEmptyValue = false; - bool res = Forward(); // TODO delete this after format change - if (res && MeasureNarrowKey() != 0) { - return res; // there was not "\0" key - } - // otherwise we are skipping "\0" key - } - - if (!Trie.Length) + TOpaqueTrieIterator::TOpaqueTrieIterator(const TOpaqueTrie& trie, const char* emptyValue, bool atend, + size_t maxKeyLength) + : Trie(trie) + , EmptyValue(emptyValue) + , AtEmptyValue(emptyValue && !atend) + , MaxKeyLength(maxKeyLength) + { + if (!AtEmptyValue && !atend) + Forward(); + } + + bool TOpaqueTrieIterator::operator==(const TOpaqueTrieIterator& rhs) const { + return (Trie == rhs.Trie && + Forks == rhs.Forks && + EmptyValue == rhs.EmptyValue && + AtEmptyValue == rhs.AtEmptyValue && + MaxKeyLength == rhs.MaxKeyLength); + } + + bool TOpaqueTrieIterator::HasMaxKeyLength() const { + return MaxKeyLength != size_t(-1) && MeasureNarrowKey() == MaxKeyLength; + } + + bool TOpaqueTrieIterator::Forward() { + if (AtEmptyValue) { + AtEmptyValue = false; + bool res = Forward(); // TODO delete this after format change + if (res && MeasureNarrowKey() != 0) { + return res; // there was not "\0" key + } + // otherwise we are skipping "\0" key + } + + if (!Trie.Length) + return false; + + if (Forks.Empty()) { + TFork fork(Trie.Data, 0, Trie.Length, Trie.SkipFunction); + Forks.Push(fork); + } else { + TFork* topFork = &Forks.Top(); + while (!topFork->NextDirection()) { + if (topFork->Node.GetOffset() >= Trie.Length) + return false; + Forks.Pop(); + if (Forks.Empty()) + return false; + topFork = &Forks.Top(); + } + } + + Y_ASSERT(!Forks.Empty()); + while (Forks.Top().CurrentDirection != D_FINAL && !HasMaxKeyLength()) { + TFork nextFork = Forks.Top().NextFork(Trie.SkipFunction); + Forks.Push(nextFork); + } + TFork& top = Forks.Top(); + static_assert(D_FINAL < D_NEXT, "relative order of NEXT and FINAL directions has changed"); + if (HasMaxKeyLength() && top.CurrentDirection == D_FINAL && top.HasDirection(D_NEXT)) { + top.NextDirection(); + } + return true; + } + + bool TOpaqueTrieIterator::Backward() { + if (AtEmptyValue) return false; - if (Forks.Empty()) { - TFork fork(Trie.Data, 0, Trie.Length, Trie.SkipFunction); - Forks.Push(fork); - } else { - TFork* topFork = &Forks.Top(); - while (!topFork->NextDirection()) { - if (topFork->Node.GetOffset() >= Trie.Length) - return false; - Forks.Pop(); - if (Forks.Empty()) - return false; - topFork = &Forks.Top(); - } - } - - Y_ASSERT(!Forks.Empty()); - while (Forks.Top().CurrentDirection != D_FINAL && !HasMaxKeyLength()) { - TFork nextFork = Forks.Top().NextFork(Trie.SkipFunction); - Forks.Push(nextFork); - } - TFork& top = Forks.Top(); - static_assert(D_FINAL < D_NEXT, "relative order of NEXT and FINAL directions has changed"); - if (HasMaxKeyLength() && top.CurrentDirection == D_FINAL && top.HasDirection(D_NEXT)) { - top.NextDirection(); - } - return true; - } - - bool TOpaqueTrieIterator::Backward() { - if (AtEmptyValue) - return false; - - if (!Trie.Length) { - if (EmptyValue) { - // A trie that has only the empty value; - // we are not at the empty value, so move to it. + if (!Trie.Length) { + if (EmptyValue) { + // A trie that has only the empty value; + // we are not at the empty value, so move to it. AtEmptyValue = true; return true; - } else { - // Empty trie. - return false; - } - } - - if (Forks.Empty()) { - TFork fork(Trie.Data, 0, Trie.Length, Trie.SkipFunction); - fork.LastDirection(); - Forks.Push(fork); - } else { - TFork* topFork = &Forks.Top(); - while (!topFork->PrevDirection()) { - if (topFork->Node.GetOffset() >= Trie.Length) - return false; - Forks.Pop(); - if (!Forks.Empty()) { - topFork = &Forks.Top(); - } else { - // When there are no more forks, - // we have to iterate over the empty value. - if (!EmptyValue) - return false; - AtEmptyValue = true; - return true; - } + } else { + // Empty trie. + return false; } } - Y_ASSERT(!Forks.Empty()); - while (Forks.Top().CurrentDirection != D_FINAL && !HasMaxKeyLength()) { - TFork nextFork = Forks.Top().NextFork(Trie.SkipFunction); - nextFork.LastDirection(); - Forks.Push(nextFork); - } - TFork& top = Forks.Top(); - static_assert(D_FINAL < D_NEXT, "relative order of NEXT and FINAL directions has changed"); - if (HasMaxKeyLength() && top.CurrentDirection == D_NEXT && top.HasDirection(D_FINAL)) { - top.PrevDirection(); - } - if (MeasureNarrowKey() == 0) { - // This is the '\0' key, skip it and get to the EmptyValue. - AtEmptyValue = true; - Forks.Clear(); - } - return true; - } - - const char* TOpaqueTrieIterator::GetValuePtr() const { - if (!Forks.Empty()) { - const TFork& lastFork = Forks.Top(); - Y_ASSERT(lastFork.Node.IsFinal() && lastFork.CurrentDirection == D_FINAL); - return Trie.Data + lastFork.GetValueOffset(); - } - Y_ASSERT(AtEmptyValue); - return EmptyValue; - } - - //------------------------------------------------------------------------- - - TString TForkStack::GetKey() const { - if (HasEmptyKey()) { - return TString(); - } - - TString result(Key); - if (TopHasLabelInKey()) { - result.append(Top().GetLabel()); - } - return result; - } - - bool TForkStack::HasEmptyKey() const { - // Special case: if we get a single zero label, treat it as an empty key - // TODO delete this after format change - if (TopHasLabelInKey()) { - return Key.size() == 0 && Top().GetLabel() == '\0'; - } else { - return Key.size() == 1 && Key[0] == '\0'; - } - } - - size_t TForkStack::MeasureKey() const { - size_t result = Key.size() + (TopHasLabelInKey() ? 1 : 0); - if (result == 1 && HasEmptyKey()) { - return 0; - } - return result; - } - - //------------------------------------------------------------------------- - - TFork::TFork(const char* data, size_t offset, size_t limit, const ILeafSkipper& skipper) - : Node(data, offset, skipper) - , Data(data) - , Limit(limit) - , CurrentDirection(TDirection(0)) - { + if (Forks.Empty()) { + TFork fork(Trie.Data, 0, Trie.Length, Trie.SkipFunction); + fork.LastDirection(); + Forks.Push(fork); + } else { + TFork* topFork = &Forks.Top(); + while (!topFork->PrevDirection()) { + if (topFork->Node.GetOffset() >= Trie.Length) + return false; + Forks.Pop(); + if (!Forks.Empty()) { + topFork = &Forks.Top(); + } else { + // When there are no more forks, + // we have to iterate over the empty value. + if (!EmptyValue) + return false; + AtEmptyValue = true; + return true; + } + } + } + + Y_ASSERT(!Forks.Empty()); + while (Forks.Top().CurrentDirection != D_FINAL && !HasMaxKeyLength()) { + TFork nextFork = Forks.Top().NextFork(Trie.SkipFunction); + nextFork.LastDirection(); + Forks.Push(nextFork); + } + TFork& top = Forks.Top(); + static_assert(D_FINAL < D_NEXT, "relative order of NEXT and FINAL directions has changed"); + if (HasMaxKeyLength() && top.CurrentDirection == D_NEXT && top.HasDirection(D_FINAL)) { + top.PrevDirection(); + } + if (MeasureNarrowKey() == 0) { + // This is the '\0' key, skip it and get to the EmptyValue. + AtEmptyValue = true; + Forks.Clear(); + } + return true; + } + + const char* TOpaqueTrieIterator::GetValuePtr() const { + if (!Forks.Empty()) { + const TFork& lastFork = Forks.Top(); + Y_ASSERT(lastFork.Node.IsFinal() && lastFork.CurrentDirection == D_FINAL); + return Trie.Data + lastFork.GetValueOffset(); + } + Y_ASSERT(AtEmptyValue); + return EmptyValue; + } + + //------------------------------------------------------------------------- + + TString TForkStack::GetKey() const { + if (HasEmptyKey()) { + return TString(); + } + + TString result(Key); + if (TopHasLabelInKey()) { + result.append(Top().GetLabel()); + } + return result; + } + + bool TForkStack::HasEmptyKey() const { + // Special case: if we get a single zero label, treat it as an empty key + // TODO delete this after format change + if (TopHasLabelInKey()) { + return Key.size() == 0 && Top().GetLabel() == '\0'; + } else { + return Key.size() == 1 && Key[0] == '\0'; + } + } + + size_t TForkStack::MeasureKey() const { + size_t result = Key.size() + (TopHasLabelInKey() ? 1 : 0); + if (result == 1 && HasEmptyKey()) { + return 0; + } + return result; + } + + //------------------------------------------------------------------------- + + TFork::TFork(const char* data, size_t offset, size_t limit, const ILeafSkipper& skipper) + : Node(data, offset, skipper) + , Data(data) + , Limit(limit) + , CurrentDirection(TDirection(0)) + { #if COMPTRIE_DATA_CHECK - if (Node.GetOffset() >= Limit - 1) - ythrow yexception() << "gone beyond the limit, data is corrupted"; + if (Node.GetOffset() >= Limit - 1) + ythrow yexception() << "gone beyond the limit, data is corrupted"; #endif - while (CurrentDirection < D_MAX && !HasDirection(CurrentDirection)) { - ++CurrentDirection; - } - } - - bool TFork::operator==(const TFork& rhs) const { - return (Data == rhs.Data && - Node.GetOffset() == rhs.Node.GetOffset() && - CurrentDirection == rhs.CurrentDirection); - } - - inline bool TFork::NextDirection() { - do { - ++CurrentDirection; - } while (CurrentDirection < D_MAX && !HasDirection(CurrentDirection)); - return CurrentDirection < D_MAX; - } - - inline bool TFork::PrevDirection() { - if (CurrentDirection == TDirection(0)) { - return false; - } - do { - --CurrentDirection; - } while (CurrentDirection > 0 && !HasDirection(CurrentDirection)); - return HasDirection(CurrentDirection); - } - - void TFork::LastDirection() { - CurrentDirection = D_MAX; - PrevDirection(); - } - - bool TFork::SetDirection(TDirection direction) { - if (!HasDirection(direction)) { - return false; - } - CurrentDirection = direction; - return true; - } - - char TFork::GetLabel() const { - return Node.GetLabel(); - } - - size_t TFork::GetValueOffset() const { - return Node.GetLeafOffset(); + while (CurrentDirection < D_MAX && !HasDirection(CurrentDirection)) { + ++CurrentDirection; + } + } + + bool TFork::operator==(const TFork& rhs) const { + return (Data == rhs.Data && + Node.GetOffset() == rhs.Node.GetOffset() && + CurrentDirection == rhs.CurrentDirection); + } + + inline bool TFork::NextDirection() { + do { + ++CurrentDirection; + } while (CurrentDirection < D_MAX && !HasDirection(CurrentDirection)); + return CurrentDirection < D_MAX; + } + + inline bool TFork::PrevDirection() { + if (CurrentDirection == TDirection(0)) { + return false; + } + do { + --CurrentDirection; + } while (CurrentDirection > 0 && !HasDirection(CurrentDirection)); + return HasDirection(CurrentDirection); + } + + void TFork::LastDirection() { + CurrentDirection = D_MAX; + PrevDirection(); + } + + bool TFork::SetDirection(TDirection direction) { + if (!HasDirection(direction)) { + return false; + } + CurrentDirection = direction; + return true; + } + + char TFork::GetLabel() const { + return Node.GetLabel(); + } + + size_t TFork::GetValueOffset() const { + return Node.GetLeafOffset(); } } diff --git a/library/cpp/containers/comptrie/opaque_trie_iterator.h b/library/cpp/containers/comptrie/opaque_trie_iterator.h index 195da3c191..8e615d7e70 100644 --- a/library/cpp/containers/comptrie/opaque_trie_iterator.h +++ b/library/cpp/containers/comptrie/opaque_trie_iterator.h @@ -9,258 +9,258 @@ #include <util/generic/yexception.h> namespace NCompactTrie { - class ILeafSkipper; - - class TFork { // Auxiliary class for a branching point in the iterator - public: - TNode Node; - const char* Data; - size_t Limit; // valid data is in range [Data + Node.GetOffset(), Data + Limit) - TDirection CurrentDirection; - - public: - TFork(const char* data, size_t offset, size_t limit, const ILeafSkipper& skipper); - - bool operator==(const TFork& rhs) const; - - bool HasLabelInKey() const { - return CurrentDirection == D_NEXT || CurrentDirection == D_FINAL; - } - - bool NextDirection(); - bool PrevDirection(); - void LastDirection(); - - bool HasDirection(TDirection direction) const { - return Node.GetOffsetByDirection(direction); - } - // If the fork doesn't have the specified direction, - // leaves the direction intact and returns false. - // Otherwise returns true. - bool SetDirection(TDirection direction); - TFork NextFork(const ILeafSkipper& skipper) const; - - char GetLabel() const; - size_t GetValueOffset() const; - }; - - inline TFork TFork::NextFork(const ILeafSkipper& skipper) const { - Y_ASSERT(CurrentDirection != D_FINAL); - size_t offset = Node.GetOffsetByDirection(CurrentDirection); - return TFork(Data, offset, Limit, skipper); + class ILeafSkipper; + + class TFork { // Auxiliary class for a branching point in the iterator + public: + TNode Node; + const char* Data; + size_t Limit; // valid data is in range [Data + Node.GetOffset(), Data + Limit) + TDirection CurrentDirection; + + public: + TFork(const char* data, size_t offset, size_t limit, const ILeafSkipper& skipper); + + bool operator==(const TFork& rhs) const; + + bool HasLabelInKey() const { + return CurrentDirection == D_NEXT || CurrentDirection == D_FINAL; + } + + bool NextDirection(); + bool PrevDirection(); + void LastDirection(); + + bool HasDirection(TDirection direction) const { + return Node.GetOffsetByDirection(direction); + } + // If the fork doesn't have the specified direction, + // leaves the direction intact and returns false. + // Otherwise returns true. + bool SetDirection(TDirection direction); + TFork NextFork(const ILeafSkipper& skipper) const; + + char GetLabel() const; + size_t GetValueOffset() const; + }; + + inline TFork TFork::NextFork(const ILeafSkipper& skipper) const { + Y_ASSERT(CurrentDirection != D_FINAL); + size_t offset = Node.GetOffsetByDirection(CurrentDirection); + return TFork(Data, offset, Limit, skipper); } - //------------------------------------------------------------------------------------------------ - class TForkStack { - public: - void Push(const TFork& fork) { - if (TopHasLabelInKey()) { - Key.push_back(Top().GetLabel()); - } - Forks.push_back(fork); - } - - void Pop() { - Forks.pop_back(); - if (TopHasLabelInKey()) { - Key.pop_back(); - } - } - - TFork& Top() { - return Forks.back(); - } - const TFork& Top() const { - return Forks.back(); - } - - bool Empty() const { - return Forks.empty(); - } - - void Clear() { - Forks.clear(); - Key.clear(); - } - - bool operator==(const TForkStack& other) const { - return Forks == other.Forks; - } - bool operator!=(const TForkStack& other) const { - return !(*this == other); - } - - TString GetKey() const; - size_t MeasureKey() const; - - private: - TVector<TFork> Forks; - TString Key; - - private: - bool TopHasLabelInKey() const { - return !Empty() && Top().HasLabelInKey(); - } - bool HasEmptyKey() const; - }; - - //------------------------------------------------------------------------------------------------ - - template <class TSymbol> - struct TConvertRawKey { - typedef typename TCompactTrieKeySelector<TSymbol>::TKey TKey; - static TKey Get(const TString& rawkey) { - TKey result; - const size_t sz = rawkey.size(); - result.reserve(sz / sizeof(TSymbol)); - for (size_t i = 0; i < sz; i += sizeof(TSymbol)) { - TSymbol sym = 0; - for (size_t j = 0; j < sizeof(TSymbol); j++) { - if (sizeof(TSymbol) <= 1) - sym = 0; - else - sym <<= 8; - if (i + j < sz) - sym |= TSymbol(0x00FF & rawkey[i + j]); - } - result.push_back(sym); - } - return result; - } - - static size_t Size(size_t rawsize) { - return rawsize / sizeof(TSymbol); - } - }; - - template <> - struct TConvertRawKey<char> { - static TString Get(const TString& rawkey) { - return rawkey; + //------------------------------------------------------------------------------------------------ + class TForkStack { + public: + void Push(const TFork& fork) { + if (TopHasLabelInKey()) { + Key.push_back(Top().GetLabel()); + } + Forks.push_back(fork); + } + + void Pop() { + Forks.pop_back(); + if (TopHasLabelInKey()) { + Key.pop_back(); + } + } + + TFork& Top() { + return Forks.back(); } + const TFork& Top() const { + return Forks.back(); + } - static size_t Size(size_t rawsize) { - return rawsize; - } - }; - - //------------------------------------------------------------------------------------------------ - class TOpaqueTrieIterator { // Iterator stuff. Stores a stack of visited forks. - public: - TOpaqueTrieIterator(const TOpaqueTrie& trie, const char* emptyValue, bool atend, - size_t maxKeyLength = size_t(-1)); - - bool operator==(const TOpaqueTrieIterator& rhs) const; - bool operator!=(const TOpaqueTrieIterator& rhs) const { - return !(*this == rhs); - } - - bool Forward(); - bool Backward(); - - template <class TSymbol> - bool UpperBound(const typename TCompactTrieKeySelector<TSymbol>::TKeyBuf& key); // True if matched exactly. + bool Empty() const { + return Forks.empty(); + } - template <class TSymbol> - typename TCompactTrieKeySelector<TSymbol>::TKey GetKey() const { - return TConvertRawKey<TSymbol>::Get(GetNarrowKey()); + void Clear() { + Forks.clear(); + Key.clear(); } - template <class TSymbol> - size_t MeasureKey() const { - return TConvertRawKey<TSymbol>::Size(MeasureNarrowKey()); + bool operator==(const TForkStack& other) const { + return Forks == other.Forks; + } + bool operator!=(const TForkStack& other) const { + return !(*this == other); + } + + TString GetKey() const; + size_t MeasureKey() const; + + private: + TVector<TFork> Forks; + TString Key; + + private: + bool TopHasLabelInKey() const { + return !Empty() && Top().HasLabelInKey(); + } + bool HasEmptyKey() const; + }; + + //------------------------------------------------------------------------------------------------ + + template <class TSymbol> + struct TConvertRawKey { + typedef typename TCompactTrieKeySelector<TSymbol>::TKey TKey; + static TKey Get(const TString& rawkey) { + TKey result; + const size_t sz = rawkey.size(); + result.reserve(sz / sizeof(TSymbol)); + for (size_t i = 0; i < sz; i += sizeof(TSymbol)) { + TSymbol sym = 0; + for (size_t j = 0; j < sizeof(TSymbol); j++) { + if (sizeof(TSymbol) <= 1) + sym = 0; + else + sym <<= 8; + if (i + j < sz) + sym |= TSymbol(0x00FF & rawkey[i + j]); + } + result.push_back(sym); + } + return result; + } + + static size_t Size(size_t rawsize) { + return rawsize / sizeof(TSymbol); + } + }; + + template <> + struct TConvertRawKey<char> { + static TString Get(const TString& rawkey) { + return rawkey; + } + + static size_t Size(size_t rawsize) { + return rawsize; } - - TString GetNarrowKey() const { - return Forks.GetKey(); - } - size_t MeasureNarrowKey() const { - return Forks.MeasureKey(); - } - - const char* GetValuePtr() const; // 0 if none - const TNode& GetNode() const { // Could be called for non-empty key and not AtEnd. - return Forks.Top().Node; - } - const TOpaqueTrie& GetTrie() const { - return Trie; - } - - private: - TOpaqueTrie Trie; - TForkStack Forks; - const char* const EmptyValue; - bool AtEmptyValue; - const size_t MaxKeyLength; - - private: - bool HasMaxKeyLength() const; - - template <class TSymbol> - int LongestPrefix(const typename TCompactTrieKeySelector<TSymbol>::TKeyBuf& key); // Used in UpperBound. - }; + }; + + //------------------------------------------------------------------------------------------------ + class TOpaqueTrieIterator { // Iterator stuff. Stores a stack of visited forks. + public: + TOpaqueTrieIterator(const TOpaqueTrie& trie, const char* emptyValue, bool atend, + size_t maxKeyLength = size_t(-1)); + + bool operator==(const TOpaqueTrieIterator& rhs) const; + bool operator!=(const TOpaqueTrieIterator& rhs) const { + return !(*this == rhs); + } + + bool Forward(); + bool Backward(); + + template <class TSymbol> + bool UpperBound(const typename TCompactTrieKeySelector<TSymbol>::TKeyBuf& key); // True if matched exactly. + + template <class TSymbol> + typename TCompactTrieKeySelector<TSymbol>::TKey GetKey() const { + return TConvertRawKey<TSymbol>::Get(GetNarrowKey()); + } + + template <class TSymbol> + size_t MeasureKey() const { + return TConvertRawKey<TSymbol>::Size(MeasureNarrowKey()); + } + + TString GetNarrowKey() const { + return Forks.GetKey(); + } + size_t MeasureNarrowKey() const { + return Forks.MeasureKey(); + } + + const char* GetValuePtr() const; // 0 if none + const TNode& GetNode() const { // Could be called for non-empty key and not AtEnd. + return Forks.Top().Node; + } + const TOpaqueTrie& GetTrie() const { + return Trie; + } + + private: + TOpaqueTrie Trie; + TForkStack Forks; + const char* const EmptyValue; + bool AtEmptyValue; + const size_t MaxKeyLength; + + private: + bool HasMaxKeyLength() const; + + template <class TSymbol> + int LongestPrefix(const typename TCompactTrieKeySelector<TSymbol>::TKeyBuf& key); // Used in UpperBound. + }; template <class TSymbol> - int TOpaqueTrieIterator::LongestPrefix(const typename TCompactTrieKeySelector<TSymbol>::TKeyBuf& key) { - Forks.Clear(); - TFork next(Trie.Data, 0, Trie.Length, Trie.SkipFunction); - for (size_t i = 0; i < key.size(); i++) { - TSymbol symbol = key[i]; - const bool isLastSymbol = (i + 1 == key.size()); - for (i64 shift = (i64)NCompactTrie::ExtraBits<TSymbol>(); shift >= 0; shift -= 8) { - const unsigned char label = (unsigned char)(symbol >> shift); - const bool isLastByte = (isLastSymbol && shift == 0); - do { - Forks.Push(next); - TFork& top = Forks.Top(); - if (label < (unsigned char)top.GetLabel()) { - if (!top.SetDirection(D_LEFT)) - return 1; - } else if (label > (unsigned char)top.GetLabel()) { - if (!top.SetDirection(D_RIGHT)) { - Forks.Pop(); // We don't pass this fork on the way to the upper bound. - return -1; - } - } else if (isLastByte) { // Here and below label == top.GetLabel(). - if (top.SetDirection(D_FINAL)) { - return 0; // Skip the NextFork() call at the end of the cycle. - } else { - top.SetDirection(D_NEXT); - return 1; - } - } else if (!top.SetDirection(D_NEXT)) { - top.SetDirection(D_FINAL); + int TOpaqueTrieIterator::LongestPrefix(const typename TCompactTrieKeySelector<TSymbol>::TKeyBuf& key) { + Forks.Clear(); + TFork next(Trie.Data, 0, Trie.Length, Trie.SkipFunction); + for (size_t i = 0; i < key.size(); i++) { + TSymbol symbol = key[i]; + const bool isLastSymbol = (i + 1 == key.size()); + for (i64 shift = (i64)NCompactTrie::ExtraBits<TSymbol>(); shift >= 0; shift -= 8) { + const unsigned char label = (unsigned char)(symbol >> shift); + const bool isLastByte = (isLastSymbol && shift == 0); + do { + Forks.Push(next); + TFork& top = Forks.Top(); + if (label < (unsigned char)top.GetLabel()) { + if (!top.SetDirection(D_LEFT)) + return 1; + } else if (label > (unsigned char)top.GetLabel()) { + if (!top.SetDirection(D_RIGHT)) { + Forks.Pop(); // We don't pass this fork on the way to the upper bound. + return -1; + } + } else if (isLastByte) { // Here and below label == top.GetLabel(). + if (top.SetDirection(D_FINAL)) { + return 0; // Skip the NextFork() call at the end of the cycle. + } else { + top.SetDirection(D_NEXT); + return 1; + } + } else if (!top.SetDirection(D_NEXT)) { + top.SetDirection(D_FINAL); return -1; } - next = top.NextFork(Trie.SkipFunction); - } while (Forks.Top().CurrentDirection != D_NEXT); // Proceed to the next byte. - } + next = top.NextFork(Trie.SkipFunction); + } while (Forks.Top().CurrentDirection != D_NEXT); // Proceed to the next byte. + } } - // We get here only if the key was empty. - Forks.Push(next); - return 1; + // We get here only if the key was empty. + Forks.Push(next); + return 1; } - template <class TSymbol> - bool TOpaqueTrieIterator::UpperBound(const typename TCompactTrieKeySelector<TSymbol>::TKeyBuf& key) { - Forks.Clear(); - if (key.empty() && EmptyValue) { - AtEmptyValue = true; - return true; - } else { - AtEmptyValue = false; - } - const int defect = LongestPrefix<TSymbol>(key); - if (defect > 0) { - // Continue the constructed forks with the smallest key possible. - while (Forks.Top().CurrentDirection != D_FINAL) { - TFork next = Forks.Top().NextFork(Trie.SkipFunction); - Forks.Push(next); - } - } else if (defect < 0) { - Forward(); + template <class TSymbol> + bool TOpaqueTrieIterator::UpperBound(const typename TCompactTrieKeySelector<TSymbol>::TKeyBuf& key) { + Forks.Clear(); + if (key.empty() && EmptyValue) { + AtEmptyValue = true; + return true; + } else { + AtEmptyValue = false; } - return defect == 0; + const int defect = LongestPrefix<TSymbol>(key); + if (defect > 0) { + // Continue the constructed forks with the smallest key possible. + while (Forks.Top().CurrentDirection != D_FINAL) { + TFork next = Forks.Top().NextFork(Trie.SkipFunction); + Forks.Push(next); + } + } else if (defect < 0) { + Forward(); + } + return defect == 0; } - + } diff --git a/library/cpp/containers/comptrie/prefix_iterator.h b/library/cpp/containers/comptrie/prefix_iterator.h index b369bb4f42..eb2b2d6878 100644 --- a/library/cpp/containers/comptrie/prefix_iterator.h +++ b/library/cpp/containers/comptrie/prefix_iterator.h @@ -1,5 +1,5 @@ #pragma once - + #include "comptrie_trie.h" // Iterates over all prefixes of the given key in the trie. @@ -39,11 +39,11 @@ public: result = Next(); } - operator bool() const { + operator bool() const { return result; } - TPrefixIterator& operator++() { + TPrefixIterator& operator++() { result = Next(); return *this; } diff --git a/library/cpp/containers/comptrie/protopacker.h b/library/cpp/containers/comptrie/protopacker.h index 3e15866dc5..5bedde270d 100644 --- a/library/cpp/containers/comptrie/protopacker.h +++ b/library/cpp/containers/comptrie/protopacker.h @@ -3,7 +3,7 @@ #include <util/stream/mem.h> #include <util/ysaveload.h> -template <class Proto> +template <class Proto> class TProtoPacker { public: TProtoPacker() = default; diff --git a/library/cpp/containers/comptrie/search_iterator.h b/library/cpp/containers/comptrie/search_iterator.h index 247f7e5936..ad6bcfdf60 100644 --- a/library/cpp/containers/comptrie/search_iterator.h +++ b/library/cpp/containers/comptrie/search_iterator.h @@ -1,5 +1,5 @@ #pragma once - + #include "comptrie_trie.h" #include "first_symbol_iterator.h" @@ -74,7 +74,7 @@ private: const char* ValuePos = nullptr; }; -template <class TTrie> +template <class TTrie> inline TSearchIterator<TTrie> MakeSearchIterator(const TTrie& trie) { return TSearchIterator<TTrie>(trie); } diff --git a/library/cpp/containers/comptrie/set.h b/library/cpp/containers/comptrie/set.h index acd43338f0..ffea92485c 100644 --- a/library/cpp/containers/comptrie/set.h +++ b/library/cpp/containers/comptrie/set.h @@ -1,16 +1,16 @@ -#pragma once - +#pragma once + #include "comptrie_trie.h" template <typename T = char> -class TCompactTrieSet: public TCompactTrie<T, ui8, TNullPacker<ui8>> { +class TCompactTrieSet: public TCompactTrie<T, ui8, TNullPacker<ui8>> { public: - typedef TCompactTrie<T, ui8, TNullPacker<ui8>> TBase; + typedef TCompactTrie<T, ui8, TNullPacker<ui8>> TBase; - using typename TBase::TBuilder; + using typename TBase::TBuilder; using typename TBase::TKey; using typename TBase::TKeyBuf; - using typename TBase::TSymbol; + using typename TBase::TSymbol; TCompactTrieSet() = default; @@ -20,8 +20,8 @@ public: } template <typename D> - explicit TCompactTrieSet(const TCompactTrie<T, D, TNullPacker<D>>& trie) - : TBase(trie.Data()) // should be binary compatible for any D + explicit TCompactTrieSet(const TCompactTrie<T, D, TNullPacker<D>>& trie) + : TBase(trie.Data()) // should be binary compatible for any D { } diff --git a/library/cpp/containers/comptrie/write_trie_backwards.cpp b/library/cpp/containers/comptrie/write_trie_backwards.cpp index fd8c28b0ed..f7459b1c49 100644 --- a/library/cpp/containers/comptrie/write_trie_backwards.cpp +++ b/library/cpp/containers/comptrie/write_trie_backwards.cpp @@ -7,104 +7,104 @@ #include <util/generic/vector.h> namespace NCompactTrie { - size_t WriteTrieBackwards(IOutputStream& os, TReverseNodeEnumerator& enumerator, bool verbose) { - if (verbose) { - Cerr << "Writing down the trie..." << Endl; - } - - // Rewrite everything from the back, removing unused pieces. - const size_t chunksize = 0x10000; - TVector<char*> resultData; - - resultData.push_back(new char[chunksize]); - char* chunkend = resultData.back() + chunksize; - - size_t resultLength = 0; - size_t chunkLength = 0; - - size_t counter = 0; - TBuffer bufferHolder; - while (enumerator.Move()) { - if (verbose) - ShowProgress(++counter); - - size_t bufferLength = 64 + enumerator.GetLeafLength(); // never know how big leaf data can be - bufferHolder.Clear(); - bufferHolder.Resize(bufferLength); - char* buffer = bufferHolder.Data(); - - size_t nodelength = enumerator.RecreateNode(buffer, resultLength); - Y_ASSERT(nodelength <= bufferLength); - - resultLength += nodelength; - - if (chunkLength + nodelength <= chunksize) { - chunkLength += nodelength; - memcpy(chunkend - chunkLength, buffer, nodelength); - } else { // allocate a new chunk - memcpy(chunkend - chunksize, buffer + nodelength - (chunksize - chunkLength), chunksize - chunkLength); - chunkLength = chunkLength + nodelength - chunksize; - - resultData.push_back(new char[chunksize]); - chunkend = resultData.back() + chunksize; - - while (chunkLength > chunksize) { // allocate a new chunks - chunkLength -= chunksize; - memcpy(chunkend - chunksize, buffer + chunkLength, chunksize); - - resultData.push_back(new char[chunksize]); - chunkend = resultData.back() + chunksize; - } - - memcpy(chunkend - chunkLength, buffer, chunkLength); + size_t WriteTrieBackwards(IOutputStream& os, TReverseNodeEnumerator& enumerator, bool verbose) { + if (verbose) { + Cerr << "Writing down the trie..." << Endl; + } + + // Rewrite everything from the back, removing unused pieces. + const size_t chunksize = 0x10000; + TVector<char*> resultData; + + resultData.push_back(new char[chunksize]); + char* chunkend = resultData.back() + chunksize; + + size_t resultLength = 0; + size_t chunkLength = 0; + + size_t counter = 0; + TBuffer bufferHolder; + while (enumerator.Move()) { + if (verbose) + ShowProgress(++counter); + + size_t bufferLength = 64 + enumerator.GetLeafLength(); // never know how big leaf data can be + bufferHolder.Clear(); + bufferHolder.Resize(bufferLength); + char* buffer = bufferHolder.Data(); + + size_t nodelength = enumerator.RecreateNode(buffer, resultLength); + Y_ASSERT(nodelength <= bufferLength); + + resultLength += nodelength; + + if (chunkLength + nodelength <= chunksize) { + chunkLength += nodelength; + memcpy(chunkend - chunkLength, buffer, nodelength); + } else { // allocate a new chunk + memcpy(chunkend - chunksize, buffer + nodelength - (chunksize - chunkLength), chunksize - chunkLength); + chunkLength = chunkLength + nodelength - chunksize; + + resultData.push_back(new char[chunksize]); + chunkend = resultData.back() + chunksize; + + while (chunkLength > chunksize) { // allocate a new chunks + chunkLength -= chunksize; + memcpy(chunkend - chunksize, buffer + chunkLength, chunksize); + + resultData.push_back(new char[chunksize]); + chunkend = resultData.back() + chunksize; + } + + memcpy(chunkend - chunkLength, buffer, chunkLength); } + } + + if (verbose) + Cerr << counter << Endl; + + // Write the whole thing down + while (!resultData.empty()) { + char* chunk = resultData.back(); + os.Write(chunk + chunksize - chunkLength, chunkLength); + chunkLength = chunksize; + delete[] chunk; + resultData.pop_back(); } - - if (verbose) - Cerr << counter << Endl; - - // Write the whole thing down - while (!resultData.empty()) { - char* chunk = resultData.back(); - os.Write(chunk + chunksize - chunkLength, chunkLength); - chunkLength = chunksize; - delete[] chunk; - resultData.pop_back(); - } - - return resultLength; + + return resultLength; } - size_t WriteTrieBackwardsNoAlloc(IOutputStream& os, TReverseNodeEnumerator& enumerator, TOpaqueTrie& trie, EMinimizeMode mode) { - char* data = const_cast<char*>(trie.Data); - char* end = data + trie.Length; - char* pos = end; - - TVector<char> buf(64); - while (enumerator.Move()) { - size_t nodeLength = enumerator.RecreateNode(nullptr, end - pos); - if (nodeLength > buf.size()) - buf.resize(nodeLength); - - size_t realLength = enumerator.RecreateNode(buf.data(), end - pos); - Y_ASSERT(realLength == nodeLength); - - pos -= nodeLength; - memcpy(pos, buf.data(), nodeLength); - } - - switch (mode) { - case MM_NOALLOC: - os.Write(pos, end - pos); - break; - case MM_INPLACE: - memmove(data, pos, end - pos); - break; - default: - Y_VERIFY(false); - } - - return end - pos; + size_t WriteTrieBackwardsNoAlloc(IOutputStream& os, TReverseNodeEnumerator& enumerator, TOpaqueTrie& trie, EMinimizeMode mode) { + char* data = const_cast<char*>(trie.Data); + char* end = data + trie.Length; + char* pos = end; + + TVector<char> buf(64); + while (enumerator.Move()) { + size_t nodeLength = enumerator.RecreateNode(nullptr, end - pos); + if (nodeLength > buf.size()) + buf.resize(nodeLength); + + size_t realLength = enumerator.RecreateNode(buf.data(), end - pos); + Y_ASSERT(realLength == nodeLength); + + pos -= nodeLength; + memcpy(pos, buf.data(), nodeLength); + } + + switch (mode) { + case MM_NOALLOC: + os.Write(pos, end - pos); + break; + case MM_INPLACE: + memmove(data, pos, end - pos); + break; + default: + Y_VERIFY(false); + } + + return end - pos; } } diff --git a/library/cpp/containers/comptrie/writeable_node.cpp b/library/cpp/containers/comptrie/writeable_node.cpp index 404003dbbd..d0e986f99a 100644 --- a/library/cpp/containers/comptrie/writeable_node.cpp +++ b/library/cpp/containers/comptrie/writeable_node.cpp @@ -3,94 +3,94 @@ #include "comptrie_impl.h" namespace NCompactTrie { - TWriteableNode::TWriteableNode() - : LeafPos(nullptr) - , LeafLength(0) - , ForwardOffset(NPOS) - , LeftOffset(NPOS) - , RightOffset(NPOS) - , Label(0) - { - } - - static size_t GetOffsetFromEnd(const TNode& node, size_t absOffset) { - return absOffset ? absOffset - node.GetOffset() - node.GetCoreLength() : NPOS; - } + TWriteableNode::TWriteableNode() + : LeafPos(nullptr) + , LeafLength(0) + , ForwardOffset(NPOS) + , LeftOffset(NPOS) + , RightOffset(NPOS) + , Label(0) + { + } - TWriteableNode::TWriteableNode(const TNode& node, const char* data) - : LeafPos(node.IsFinal() ? data + node.GetLeafOffset() : nullptr) - , LeafLength(node.GetLeafLength()) - , ForwardOffset(GetOffsetFromEnd(node, node.GetForwardOffset())) - , LeftOffset(GetOffsetFromEnd(node, node.GetLeftOffset())) - , RightOffset(GetOffsetFromEnd(node, node.GetRightOffset())) - , Label(node.GetLabel()) - { - } + static size_t GetOffsetFromEnd(const TNode& node, size_t absOffset) { + return absOffset ? absOffset - node.GetOffset() - node.GetCoreLength() : NPOS; + } + + TWriteableNode::TWriteableNode(const TNode& node, const char* data) + : LeafPos(node.IsFinal() ? data + node.GetLeafOffset() : nullptr) + , LeafLength(node.GetLeafLength()) + , ForwardOffset(GetOffsetFromEnd(node, node.GetForwardOffset())) + , LeftOffset(GetOffsetFromEnd(node, node.GetLeftOffset())) + , RightOffset(GetOffsetFromEnd(node, node.GetRightOffset())) + , Label(node.GetLabel()) + { + } - size_t TWriteableNode::Measure() const { - size_t len = 2 + LeafLength; - size_t fwdLen = 0; - size_t lastLen = 0; - size_t lastFwdLen = 0; - // Now, increase all the offsets by the length and recalculate everything, until it converges - do { - lastLen = len; - lastFwdLen = fwdLen; + size_t TWriteableNode::Measure() const { + size_t len = 2 + LeafLength; + size_t fwdLen = 0; + size_t lastLen = 0; + size_t lastFwdLen = 0; + // Now, increase all the offsets by the length and recalculate everything, until it converges + do { + lastLen = len; + lastFwdLen = fwdLen; - len = 2 + LeafLength; - len += MeasureOffset(LeftOffset != NPOS ? LeftOffset + lastLen : 0); - len += MeasureOffset(RightOffset != NPOS ? RightOffset + lastLen : 0); + len = 2 + LeafLength; + len += MeasureOffset(LeftOffset != NPOS ? LeftOffset + lastLen : 0); + len += MeasureOffset(RightOffset != NPOS ? RightOffset + lastLen : 0); + + // Relative forward offset of 0 means we don't need extra length for an epsilon link. + // But an epsilon link means we need an extra 1 for the flags and the forward offset is measured + // from the start of the epsilon link, not from the start of our node. + if (ForwardOffset != NPOS && ForwardOffset != 0) { + fwdLen = MeasureOffset(ForwardOffset + lastFwdLen) + 1; + len += fwdLen; + } - // Relative forward offset of 0 means we don't need extra length for an epsilon link. - // But an epsilon link means we need an extra 1 for the flags and the forward offset is measured - // from the start of the epsilon link, not from the start of our node. - if (ForwardOffset != NPOS && ForwardOffset != 0) { - fwdLen = MeasureOffset(ForwardOffset + lastFwdLen) + 1; - len += fwdLen; - } - - } while (lastLen != len || lastFwdLen != fwdLen); - - return len; - } + } while (lastLen != len || lastFwdLen != fwdLen); - size_t TWriteableNode::Pack(char* buffer) const { - const size_t length = Measure(); + return len; + } - char flags = 0; - if (LeafPos) { - flags |= MT_FINAL; - } - if (ForwardOffset != NPOS) { - flags |= MT_NEXT; - } + size_t TWriteableNode::Pack(char* buffer) const { + const size_t length = Measure(); - const size_t leftOffset = LeftOffset != NPOS ? LeftOffset + length : 0; - const size_t rightOffset = RightOffset != NPOS ? RightOffset + length : 0; - const size_t leftOffsetSize = MeasureOffset(leftOffset); - const size_t rightOffsetSize = MeasureOffset(rightOffset); - flags |= (leftOffsetSize << MT_LEFTSHIFT); - flags |= (rightOffsetSize << MT_RIGHTSHIFT); + char flags = 0; + if (LeafPos) { + flags |= MT_FINAL; + } + if (ForwardOffset != NPOS) { + flags |= MT_NEXT; + } - buffer[0] = flags; - buffer[1] = Label; - size_t usedLen = 2; - usedLen += PackOffset(buffer + usedLen, leftOffset); - usedLen += PackOffset(buffer + usedLen, rightOffset); + const size_t leftOffset = LeftOffset != NPOS ? LeftOffset + length : 0; + const size_t rightOffset = RightOffset != NPOS ? RightOffset + length : 0; + const size_t leftOffsetSize = MeasureOffset(leftOffset); + const size_t rightOffsetSize = MeasureOffset(rightOffset); + flags |= (leftOffsetSize << MT_LEFTSHIFT); + flags |= (rightOffsetSize << MT_RIGHTSHIFT); - if (LeafPos && LeafLength) { - memcpy(buffer + usedLen, LeafPos, LeafLength); - usedLen += LeafLength; - } + buffer[0] = flags; + buffer[1] = Label; + size_t usedLen = 2; + usedLen += PackOffset(buffer + usedLen, leftOffset); + usedLen += PackOffset(buffer + usedLen, rightOffset); - if (ForwardOffset != NPOS && ForwardOffset != 0) { - const size_t fwdOffset = ForwardOffset + length - usedLen; - size_t fwdOffsetSize = MeasureOffset(fwdOffset); - buffer[usedLen++] = (char)(fwdOffsetSize & MT_SIZEMASK); - usedLen += PackOffset(buffer + usedLen, fwdOffset); - } - Y_ASSERT(usedLen == length); - return usedLen; + if (LeafPos && LeafLength) { + memcpy(buffer + usedLen, LeafPos, LeafLength); + usedLen += LeafLength; + } + + if (ForwardOffset != NPOS && ForwardOffset != 0) { + const size_t fwdOffset = ForwardOffset + length - usedLen; + size_t fwdOffsetSize = MeasureOffset(fwdOffset); + buffer[usedLen++] = (char)(fwdOffsetSize & MT_SIZEMASK); + usedLen += PackOffset(buffer + usedLen, fwdOffset); + } + Y_ASSERT(usedLen == length); + return usedLen; } } diff --git a/library/cpp/containers/comptrie/writeable_node.h b/library/cpp/containers/comptrie/writeable_node.h index 5454e579ef..585ef0a5e4 100644 --- a/library/cpp/containers/comptrie/writeable_node.h +++ b/library/cpp/containers/comptrie/writeable_node.h @@ -3,24 +3,24 @@ #include <cstddef> namespace NCompactTrie { - class TNode; + class TNode; - class TWriteableNode { - public: - const char* LeafPos; - size_t LeafLength; + class TWriteableNode { + public: + const char* LeafPos; + size_t LeafLength; - size_t ForwardOffset; - size_t LeftOffset; - size_t RightOffset; - char Label; + size_t ForwardOffset; + size_t LeftOffset; + size_t RightOffset; + char Label; - TWriteableNode(); - TWriteableNode(const TNode& node, const char* data); + TWriteableNode(); + TWriteableNode(const TNode& node, const char* data); - // When you call this, the offsets should be relative to the end of the node. Use NPOS to indicate an absent offset. - size_t Pack(char* buffer) const; - size_t Measure() const; - }; + // When you call this, the offsets should be relative to the end of the node. Use NPOS to indicate an absent offset. + size_t Pack(char* buffer) const; + size_t Measure() const; + }; } diff --git a/library/cpp/containers/comptrie/ya.make b/library/cpp/containers/comptrie/ya.make index 81352da4b2..43d6534634 100644 --- a/library/cpp/containers/comptrie/ya.make +++ b/library/cpp/containers/comptrie/ya.make @@ -1,4 +1,4 @@ -LIBRARY() +LIBRARY() OWNER(velavokr) @@ -12,9 +12,9 @@ SRCS( key_selector.h leaf_skipper.h set.h - comptrie.cpp + comptrie.cpp comptrie_builder.cpp - comptrie_impl.cpp + comptrie_impl.cpp make_fast_layout.cpp minimize.cpp node.cpp diff --git a/library/cpp/containers/intrusive_avl_tree/avltree.cpp b/library/cpp/containers/intrusive_avl_tree/avltree.cpp index dd27c7df41..05cbea330c 100644 --- a/library/cpp/containers/intrusive_avl_tree/avltree.cpp +++ b/library/cpp/containers/intrusive_avl_tree/avltree.cpp @@ -1 +1 @@ -#include "avltree.h" +#include "avltree.h" diff --git a/library/cpp/containers/intrusive_avl_tree/avltree.h b/library/cpp/containers/intrusive_avl_tree/avltree.h index a58c63b07c..596917a0e2 100644 --- a/library/cpp/containers/intrusive_avl_tree/avltree.h +++ b/library/cpp/containers/intrusive_avl_tree/avltree.h @@ -1,158 +1,158 @@ #pragma once - -#include <util/generic/noncopyable.h> - -template <class T, class C> -struct TAvlTreeItem; - -template <class T, class C> -class TAvlTree: public TNonCopyable { + +#include <util/generic/noncopyable.h> + +template <class T, class C> +struct TAvlTreeItem; + +template <class T, class C> +class TAvlTree: public TNonCopyable { using TTreeItem = TAvlTreeItem<T, C>; - friend struct TAvlTreeItem<T, C>; - + friend struct TAvlTreeItem<T, C>; + static inline const T* AsT(const TTreeItem* item) noexcept { return (const T*)item; } static inline T* AsT(TTreeItem* item) noexcept { - return (T*)item; - } - + return (T*)item; + } + template <class TTreeItem, class TValue> class TIteratorBase { - public: + public: inline TIteratorBase(TTreeItem* p, const TAvlTree* t) noexcept - : Ptr_(p) - , Tree_(t) - { - } - + : Ptr_(p) + , Tree_(t) + { + } + inline bool IsEnd() const noexcept { return Ptr_ == nullptr; - } - + } + inline bool IsBegin() const noexcept { return Ptr_ == nullptr; - } - + } + inline bool IsFirst() const noexcept { - return Ptr_ && Ptr_ == Tree_->Head_; - } - + return Ptr_ && Ptr_ == Tree_->Head_; + } + inline bool IsLast() const noexcept { - return Ptr_ && Ptr_ == Tree_->Tail_; - } - + return Ptr_ && Ptr_ == Tree_->Tail_; + } + inline TValue& operator*() const noexcept { - return *AsT(Ptr_); - } - + return *AsT(Ptr_); + } + inline TValue* operator->() const noexcept { - return AsT(Ptr_); - } - + return AsT(Ptr_); + } + inline TTreeItem* Inc() noexcept { - return Ptr_ = FindNext(Ptr_); - } - + return Ptr_ = FindNext(Ptr_); + } + inline TTreeItem* Dec() noexcept { - return Ptr_ = FindPrev(Ptr_); - } - + return Ptr_ = FindPrev(Ptr_); + } + inline TIteratorBase& operator++() noexcept { - Inc(); - return *this; - } - + Inc(); + return *this; + } + inline TIteratorBase operator++(int) noexcept { TIteratorBase ret(*this); - Inc(); - return ret; - } - + Inc(); + return ret; + } + inline TIteratorBase& operator--() noexcept { - Dec(); - return *this; - } - + Dec(); + return *this; + } + inline TIteratorBase operator--(int) noexcept { TIteratorBase ret(*this); - Dec(); - return ret; - } - + Dec(); + return ret; + } + inline TIteratorBase Next() const noexcept { - return ConstructNext(*this); - } - + return ConstructNext(*this); + } + inline TIteratorBase Prev() const noexcept { - return ConstructPrev(*this); - } - + return ConstructPrev(*this); + } + inline bool operator==(const TIteratorBase& r) const noexcept { - return Ptr_ == r.Ptr_; - } - + return Ptr_ == r.Ptr_; + } + inline bool operator!=(const TIteratorBase& r) const noexcept { - return Ptr_ != r.Ptr_; - } - - private: + return Ptr_ != r.Ptr_; + } + + private: inline static TIteratorBase ConstructNext(const TIteratorBase& i) noexcept { - return TIterator(FindNext(i.Ptr_), i.Tree_); - } - + return TIterator(FindNext(i.Ptr_), i.Tree_); + } + inline static TIteratorBase ConstructPrev(const TIteratorBase& i) noexcept { - return TIterator(FindPrev(i.Ptr_), i.Tree_); - } - + return TIterator(FindPrev(i.Ptr_), i.Tree_); + } + inline static TIteratorBase FindPrev(TTreeItem* el) noexcept { if (el->Left_ != nullptr) { - el = el->Left_; - + el = el->Left_; + while (el->Right_ != nullptr) { - el = el->Right_; - } - } else { - while (true) { - TTreeItem* last = el; - el = el->Parent_; - + el = el->Right_; + } + } else { + while (true) { + TTreeItem* last = el; + el = el->Parent_; + if (el == nullptr || el->Right_ == last) { - break; - } - } - } - - return el; - } - - static TTreeItem* FindNext(TTreeItem* el) { + break; + } + } + } + + return el; + } + + static TTreeItem* FindNext(TTreeItem* el) { if (el->Right_ != nullptr) { - el = el->Right_; - - while (el->Left_) { - el = el->Left_; - } - } else { - while (true) { - TTreeItem* last = el; - el = el->Parent_; - + el = el->Right_; + + while (el->Left_) { + el = el->Left_; + } + } else { + while (true) { + TTreeItem* last = el; + el = el->Parent_; + if (el == nullptr || el->Left_ == last) { - break; - } - } - } - - return el; - } - - private: - TTreeItem* Ptr_; - const TAvlTree* Tree_; - }; - + break; + } + } + } + + return el; + } + + private: + TTreeItem* Ptr_; + const TAvlTree* Tree_; + }; + using TConstIterator = TIteratorBase<const TTreeItem, const T>; using TIterator = TIteratorBase<TTreeItem, T>; @@ -161,222 +161,222 @@ class TAvlTree: public TNonCopyable { } static inline TIterator ConstructFirst(const TAvlTree* t) noexcept { - return TIterator(t->Head_, t); - } - + return TIterator(t->Head_, t); + } + static inline TConstIterator ConstructLastConst(const TAvlTree* t) noexcept { return TConstIterator(t->Tail_, t); } static inline TIterator ConstructLast(const TAvlTree* t) noexcept { - return TIterator(t->Tail_, t); - } - - static inline bool Compare(const TTreeItem& l, const TTreeItem& r) { - return C::Compare(*AsT(&l), *AsT(&r)); - } - -public: + return TIterator(t->Tail_, t); + } + + static inline bool Compare(const TTreeItem& l, const TTreeItem& r) { + return C::Compare(*AsT(&l), *AsT(&r)); + } + +public: using const_iterator = TConstIterator; using iterator = TIterator; - + inline TAvlTree() noexcept - : Root_(nullptr) - , Head_(nullptr) - , Tail_(nullptr) - { - } - + : Root_(nullptr) + , Head_(nullptr) + , Tail_(nullptr) + { + } + inline ~TAvlTree() noexcept { Clear(); } inline void Clear() noexcept { - for (iterator it = Begin(); it != End();) { - (it++)->TTreeItem::Unlink(); - } - } - + for (iterator it = Begin(); it != End();) { + (it++)->TTreeItem::Unlink(); + } + } + inline T* Insert(TTreeItem* el, TTreeItem** lastFound = nullptr) noexcept { - el->Unlink(); - el->Tree_ = this; - - TTreeItem* curEl = Root_; + el->Unlink(); + el->Tree_ = this; + + TTreeItem* curEl = Root_; TTreeItem* parentEl = nullptr; TTreeItem* lastLess = nullptr; - - while (true) { + + while (true) { if (curEl == nullptr) { - AttachRebal(el, parentEl, lastLess); - + AttachRebal(el, parentEl, lastLess); + if (lastFound != nullptr) { - *lastFound = el; - } - - return AsT(el); - } - - if (Compare(*el, *curEl)) { - parentEl = lastLess = curEl; - curEl = curEl->Left_; - } else if (Compare(*curEl, *el)) { - parentEl = curEl; - curEl = curEl->Right_; - } else { + *lastFound = el; + } + + return AsT(el); + } + + if (Compare(*el, *curEl)) { + parentEl = lastLess = curEl; + curEl = curEl->Left_; + } else if (Compare(*curEl, *el)) { + parentEl = curEl; + curEl = curEl->Right_; + } else { if (lastFound != nullptr) { - *lastFound = curEl; - } - + *lastFound = curEl; + } + return nullptr; - } - } - } - + } + } + } + inline T* Find(const TTreeItem* el) const noexcept { - TTreeItem* curEl = Root_; - - while (curEl) { - if (Compare(*el, *curEl)) { - curEl = curEl->Left_; - } else if (Compare(*curEl, *el)) { - curEl = curEl->Right_; - } else { - return AsT(curEl); - } - } - + TTreeItem* curEl = Root_; + + while (curEl) { + if (Compare(*el, *curEl)) { + curEl = curEl->Left_; + } else if (Compare(*curEl, *el)) { + curEl = curEl->Right_; + } else { + return AsT(curEl); + } + } + return nullptr; - } + } inline T* LowerBound(const TTreeItem* el) const noexcept { - TTreeItem* curEl = Root_; + TTreeItem* curEl = Root_; TTreeItem* lowerBound = nullptr; - - while (curEl) { - if (Compare(*el, *curEl)) { - lowerBound = curEl; - curEl = curEl->Left_; - } else if (Compare(*curEl, *el)) { - curEl = curEl->Right_; - } else { - return AsT(curEl); + + while (curEl) { + if (Compare(*el, *curEl)) { + lowerBound = curEl; + curEl = curEl->Left_; + } else if (Compare(*curEl, *el)) { + curEl = curEl->Right_; + } else { + return AsT(curEl); } } - return AsT(lowerBound); - } - + return AsT(lowerBound); + } + inline T* Erase(TTreeItem* el) noexcept { - if (el->Tree_ == this) { - return this->EraseImpl(el); - } - + if (el->Tree_ == this) { + return this->EraseImpl(el); + } + return nullptr; - } - + } + inline T* EraseImpl(TTreeItem* el) noexcept { el->Tree_ = nullptr; - - TTreeItem* replacement; - TTreeItem* fixfrom; - long lheight, rheight; - - if (el->Right_) { - replacement = el->Right_; - - while (replacement->Left_) { - replacement = replacement->Left_; - } - - if (replacement->Parent_ == el) { - fixfrom = replacement; - } else { - fixfrom = replacement->Parent_; - } - - if (el == Head_) { - Head_ = replacement; - } - - RemoveEl(replacement, replacement->Right_); - ReplaceEl(el, replacement); - } else if (el->Left_) { - replacement = el->Left_; - - while (replacement->Right_) { - replacement = replacement->Right_; - } - - if (replacement->Parent_ == el) { - fixfrom = replacement; - } else { - fixfrom = replacement->Parent_; - } - - if (el == Tail_) { - Tail_ = replacement; - } - - RemoveEl(replacement, replacement->Left_); - ReplaceEl(el, replacement); - } else { - fixfrom = el->Parent_; - - if (el == Head_) { - Head_ = el->Parent_; - } - - if (el == Tail_) { - Tail_ = el->Parent_; - } - + + TTreeItem* replacement; + TTreeItem* fixfrom; + long lheight, rheight; + + if (el->Right_) { + replacement = el->Right_; + + while (replacement->Left_) { + replacement = replacement->Left_; + } + + if (replacement->Parent_ == el) { + fixfrom = replacement; + } else { + fixfrom = replacement->Parent_; + } + + if (el == Head_) { + Head_ = replacement; + } + + RemoveEl(replacement, replacement->Right_); + ReplaceEl(el, replacement); + } else if (el->Left_) { + replacement = el->Left_; + + while (replacement->Right_) { + replacement = replacement->Right_; + } + + if (replacement->Parent_ == el) { + fixfrom = replacement; + } else { + fixfrom = replacement->Parent_; + } + + if (el == Tail_) { + Tail_ = replacement; + } + + RemoveEl(replacement, replacement->Left_); + ReplaceEl(el, replacement); + } else { + fixfrom = el->Parent_; + + if (el == Head_) { + Head_ = el->Parent_; + } + + if (el == Tail_) { + Tail_ = el->Parent_; + } + RemoveEl(el, nullptr); - } - + } + if (fixfrom == nullptr) { - return AsT(el); - } - - RecalcHeights(fixfrom); - - TTreeItem* ub = FindFirstUnbalEl(fixfrom); - - while (ub) { - lheight = ub->Left_ ? ub->Left_->Height_ : 0; - rheight = ub->Right_ ? ub->Right_->Height_ : 0; - - if (rheight > lheight) { - ub = ub->Right_; - lheight = ub->Left_ ? ub->Left_->Height_ : 0; - rheight = ub->Right_ ? ub->Right_->Height_ : 0; - - if (rheight > lheight) { - ub = ub->Right_; - } else if (rheight < lheight) { - ub = ub->Left_; - } else { - ub = ub->Right_; - } - } else { - ub = ub->Left_; - lheight = ub->Left_ ? ub->Left_->Height_ : 0; - rheight = ub->Right_ ? ub->Right_->Height_ : 0; - if (rheight > lheight) { - ub = ub->Right_; - } else if (rheight < lheight) { - ub = ub->Left_; - } else { - ub = ub->Left_; - } - } - - fixfrom = Rebalance(ub); - ub = FindFirstUnbalEl(fixfrom); - } - - return AsT(el); - } - + return AsT(el); + } + + RecalcHeights(fixfrom); + + TTreeItem* ub = FindFirstUnbalEl(fixfrom); + + while (ub) { + lheight = ub->Left_ ? ub->Left_->Height_ : 0; + rheight = ub->Right_ ? ub->Right_->Height_ : 0; + + if (rheight > lheight) { + ub = ub->Right_; + lheight = ub->Left_ ? ub->Left_->Height_ : 0; + rheight = ub->Right_ ? ub->Right_->Height_ : 0; + + if (rheight > lheight) { + ub = ub->Right_; + } else if (rheight < lheight) { + ub = ub->Left_; + } else { + ub = ub->Right_; + } + } else { + ub = ub->Left_; + lheight = ub->Left_ ? ub->Left_->Height_ : 0; + rheight = ub->Right_ ? ub->Right_->Height_ : 0; + if (rheight > lheight) { + ub = ub->Right_; + } else if (rheight < lheight) { + ub = ub->Left_; + } else { + ub = ub->Left_; + } + } + + fixfrom = Rebalance(ub); + ub = FindFirstUnbalEl(fixfrom); + } + + return AsT(el); + } + inline const_iterator First() const noexcept { return ConstructFirstConst(this); } @@ -410,21 +410,21 @@ public: } inline iterator First() noexcept { - return ConstructFirst(this); - } - + return ConstructFirst(this); + } + inline iterator Last() noexcept { - return ConstructLast(this); - } - + return ConstructLast(this); + } + inline iterator Begin() noexcept { - return First(); - } - + return First(); + } + inline iterator End() noexcept { return iterator(nullptr, this); - } - + } + inline iterator begin() noexcept { return Begin(); } @@ -434,321 +434,321 @@ public: } inline bool Empty() const noexcept { - return const_cast<TAvlTree*>(this)->Begin() == const_cast<TAvlTree*>(this)->End(); - } - + return const_cast<TAvlTree*>(this)->Begin() == const_cast<TAvlTree*>(this)->End(); + } + inline explicit operator bool() const noexcept { - return !this->Empty(); - } - - template <class Functor> - inline void ForEach(Functor& f) { - iterator it = Begin(); - - while (!it.IsEnd()) { - iterator next = it; - ++next; - f(*it); - it = next; - } - } - -private: + return !this->Empty(); + } + + template <class Functor> + inline void ForEach(Functor& f) { + iterator it = Begin(); + + while (!it.IsEnd()) { + iterator next = it; + ++next; + f(*it); + it = next; + } + } + +private: inline TTreeItem* Rebalance(TTreeItem* n) noexcept { - long lheight, rheight; - - TTreeItem* a; - TTreeItem* b; - TTreeItem* c; - TTreeItem* t1; - TTreeItem* t2; - TTreeItem* t3; - TTreeItem* t4; - - TTreeItem* p = n->Parent_; - TTreeItem* gp = p->Parent_; - TTreeItem* ggp = gp->Parent_; - - if (gp->Right_ == p) { - if (p->Right_ == n) { - a = gp; - b = p; - c = n; - t1 = gp->Left_; - t2 = p->Left_; - t3 = n->Left_; - t4 = n->Right_; - } else { - a = gp; - b = n; - c = p; - t1 = gp->Left_; - t2 = n->Left_; - t3 = n->Right_; - t4 = p->Right_; - } - } else { - if (p->Right_ == n) { - a = p; - b = n; - c = gp; - t1 = p->Left_; - t2 = n->Left_; - t3 = n->Right_; - t4 = gp->Right_; - } else { - a = n; - b = p; - c = gp; - t1 = n->Left_; - t2 = n->Right_; - t3 = p->Right_; - t4 = gp->Right_; - } - } - + long lheight, rheight; + + TTreeItem* a; + TTreeItem* b; + TTreeItem* c; + TTreeItem* t1; + TTreeItem* t2; + TTreeItem* t3; + TTreeItem* t4; + + TTreeItem* p = n->Parent_; + TTreeItem* gp = p->Parent_; + TTreeItem* ggp = gp->Parent_; + + if (gp->Right_ == p) { + if (p->Right_ == n) { + a = gp; + b = p; + c = n; + t1 = gp->Left_; + t2 = p->Left_; + t3 = n->Left_; + t4 = n->Right_; + } else { + a = gp; + b = n; + c = p; + t1 = gp->Left_; + t2 = n->Left_; + t3 = n->Right_; + t4 = p->Right_; + } + } else { + if (p->Right_ == n) { + a = p; + b = n; + c = gp; + t1 = p->Left_; + t2 = n->Left_; + t3 = n->Right_; + t4 = gp->Right_; + } else { + a = n; + b = p; + c = gp; + t1 = n->Left_; + t2 = n->Right_; + t3 = p->Right_; + t4 = gp->Right_; + } + } + if (ggp == nullptr) { - Root_ = b; - } else if (ggp->Left_ == gp) { - ggp->Left_ = b; - } else { - ggp->Right_ = b; - } - - b->Parent_ = ggp; - b->Left_ = a; - a->Parent_ = b; - b->Right_ = c; - c->Parent_ = b; - a->Left_ = t1; - + Root_ = b; + } else if (ggp->Left_ == gp) { + ggp->Left_ = b; + } else { + ggp->Right_ = b; + } + + b->Parent_ = ggp; + b->Left_ = a; + a->Parent_ = b; + b->Right_ = c; + c->Parent_ = b; + a->Left_ = t1; + if (t1 != nullptr) { - t1->Parent_ = a; - } - - a->Right_ = t2; - + t1->Parent_ = a; + } + + a->Right_ = t2; + if (t2 != nullptr) { - t2->Parent_ = a; - } - - c->Left_ = t3; - + t2->Parent_ = a; + } + + c->Left_ = t3; + if (t3 != nullptr) { - t3->Parent_ = c; - } - - c->Right_ = t4; - + t3->Parent_ = c; + } + + c->Right_ = t4; + if (t4 != nullptr) { - t4->Parent_ = c; - } - - lheight = a->Left_ ? a->Left_->Height_ : 0; - rheight = a->Right_ ? a->Right_->Height_ : 0; - a->Height_ = (lheight > rheight ? lheight : rheight) + 1; - - lheight = c->Left_ ? c->Left_->Height_ : 0; - rheight = c->Right_ ? c->Right_->Height_ : 0; - c->Height_ = (lheight > rheight ? lheight : rheight) + 1; - - lheight = a->Height_; - rheight = c->Height_; - b->Height_ = (lheight > rheight ? lheight : rheight) + 1; - - RecalcHeights(ggp); - - return ggp; - } - + t4->Parent_ = c; + } + + lheight = a->Left_ ? a->Left_->Height_ : 0; + rheight = a->Right_ ? a->Right_->Height_ : 0; + a->Height_ = (lheight > rheight ? lheight : rheight) + 1; + + lheight = c->Left_ ? c->Left_->Height_ : 0; + rheight = c->Right_ ? c->Right_->Height_ : 0; + c->Height_ = (lheight > rheight ? lheight : rheight) + 1; + + lheight = a->Height_; + rheight = c->Height_; + b->Height_ = (lheight > rheight ? lheight : rheight) + 1; + + RecalcHeights(ggp); + + return ggp; + } + inline void RecalcHeights(TTreeItem* el) noexcept { - long lheight, rheight, new_height; - - while (el) { - lheight = el->Left_ ? el->Left_->Height_ : 0; - rheight = el->Right_ ? el->Right_->Height_ : 0; - - new_height = (lheight > rheight ? lheight : rheight) + 1; - - if (new_height == el->Height_) { - return; - } else { - el->Height_ = new_height; - } - - el = el->Parent_; - } - } - + long lheight, rheight, new_height; + + while (el) { + lheight = el->Left_ ? el->Left_->Height_ : 0; + rheight = el->Right_ ? el->Right_->Height_ : 0; + + new_height = (lheight > rheight ? lheight : rheight) + 1; + + if (new_height == el->Height_) { + return; + } else { + el->Height_ = new_height; + } + + el = el->Parent_; + } + } + inline TTreeItem* FindFirstUnbalGP(TTreeItem* el) noexcept { - long lheight, rheight, balanceProp; - TTreeItem* gp; - + long lheight, rheight, balanceProp; + TTreeItem* gp; + if (el == nullptr || el->Parent_ == nullptr || el->Parent_->Parent_ == nullptr) { return nullptr; - } - - gp = el->Parent_->Parent_; - + } + + gp = el->Parent_->Parent_; + while (gp != nullptr) { - lheight = gp->Left_ ? gp->Left_->Height_ : 0; - rheight = gp->Right_ ? gp->Right_->Height_ : 0; - balanceProp = lheight - rheight; - - if (balanceProp < -1 || balanceProp > 1) { - return el; - } - - el = el->Parent_; - gp = gp->Parent_; - } - + lheight = gp->Left_ ? gp->Left_->Height_ : 0; + rheight = gp->Right_ ? gp->Right_->Height_ : 0; + balanceProp = lheight - rheight; + + if (balanceProp < -1 || balanceProp > 1) { + return el; + } + + el = el->Parent_; + gp = gp->Parent_; + } + return nullptr; - } - + } + inline TTreeItem* FindFirstUnbalEl(TTreeItem* el) noexcept { if (el == nullptr) { return nullptr; - } - - while (el) { - const long lheight = el->Left_ ? el->Left_->Height_ : 0; - const long rheight = el->Right_ ? el->Right_->Height_ : 0; - const long balanceProp = lheight - rheight; - - if (balanceProp < -1 || balanceProp > 1) { - return el; - } - - el = el->Parent_; - } - + } + + while (el) { + const long lheight = el->Left_ ? el->Left_->Height_ : 0; + const long rheight = el->Right_ ? el->Right_->Height_ : 0; + const long balanceProp = lheight - rheight; + + if (balanceProp < -1 || balanceProp > 1) { + return el; + } + + el = el->Parent_; + } + return nullptr; - } - + } + inline void ReplaceEl(TTreeItem* el, TTreeItem* replacement) noexcept { - TTreeItem* parent = el->Parent_; - TTreeItem* left = el->Left_; - TTreeItem* right = el->Right_; - - replacement->Left_ = left; - - if (left) { - left->Parent_ = replacement; - } - - replacement->Right_ = right; - - if (right) { - right->Parent_ = replacement; - } - - replacement->Parent_ = parent; - - if (parent) { - if (parent->Left_ == el) { - parent->Left_ = replacement; - } else { - parent->Right_ = replacement; - } - } else { - Root_ = replacement; - } - - replacement->Height_ = el->Height_; - } - + TTreeItem* parent = el->Parent_; + TTreeItem* left = el->Left_; + TTreeItem* right = el->Right_; + + replacement->Left_ = left; + + if (left) { + left->Parent_ = replacement; + } + + replacement->Right_ = right; + + if (right) { + right->Parent_ = replacement; + } + + replacement->Parent_ = parent; + + if (parent) { + if (parent->Left_ == el) { + parent->Left_ = replacement; + } else { + parent->Right_ = replacement; + } + } else { + Root_ = replacement; + } + + replacement->Height_ = el->Height_; + } + inline void RemoveEl(TTreeItem* el, TTreeItem* filler) noexcept { - TTreeItem* parent = el->Parent_; - - if (parent) { - if (parent->Left_ == el) { - parent->Left_ = filler; - } else { - parent->Right_ = filler; - } - } else { - Root_ = filler; - } - - if (filler) { - filler->Parent_ = parent; - } - - return; - } - - inline void AttachRebal(TTreeItem* el, TTreeItem* parentEl, TTreeItem* lastLess) { - el->Parent_ = parentEl; + TTreeItem* parent = el->Parent_; + + if (parent) { + if (parent->Left_ == el) { + parent->Left_ = filler; + } else { + parent->Right_ = filler; + } + } else { + Root_ = filler; + } + + if (filler) { + filler->Parent_ = parent; + } + + return; + } + + inline void AttachRebal(TTreeItem* el, TTreeItem* parentEl, TTreeItem* lastLess) { + el->Parent_ = parentEl; el->Left_ = nullptr; el->Right_ = nullptr; - el->Height_ = 1; - + el->Height_ = 1; + if (parentEl != nullptr) { - if (lastLess == parentEl) { - parentEl->Left_ = el; - } else { - parentEl->Right_ = el; - } - - if (Head_->Left_ == el) { - Head_ = el; - } - - if (Tail_->Right_ == el) { - Tail_ = el; - } - } else { - Root_ = el; - Head_ = Tail_ = el; - } - - RecalcHeights(parentEl); - - TTreeItem* ub = FindFirstUnbalGP(el); - + if (lastLess == parentEl) { + parentEl->Left_ = el; + } else { + parentEl->Right_ = el; + } + + if (Head_->Left_ == el) { + Head_ = el; + } + + if (Tail_->Right_ == el) { + Tail_ = el; + } + } else { + Root_ = el; + Head_ = Tail_ = el; + } + + RecalcHeights(parentEl); + + TTreeItem* ub = FindFirstUnbalGP(el); + if (ub != nullptr) { - Rebalance(ub); - } - } - -private: - TTreeItem* Root_; - TTreeItem* Head_; - TTreeItem* Tail_; -}; - -template <class T, class C> -struct TAvlTreeItem: public TNonCopyable { -public: + Rebalance(ub); + } + } + +private: + TTreeItem* Root_; + TTreeItem* Head_; + TTreeItem* Tail_; +}; + +template <class T, class C> +struct TAvlTreeItem: public TNonCopyable { +public: using TTree = TAvlTree<T, C>; - friend class TAvlTree<T, C>; + friend class TAvlTree<T, C>; friend typename TAvlTree<T, C>::TConstIterator; friend typename TAvlTree<T, C>::TIterator; - + inline TAvlTreeItem() noexcept - : Left_(nullptr) - , Right_(nullptr) - , Parent_(nullptr) - , Height_(0) - , Tree_(nullptr) - { - } - + : Left_(nullptr) + , Right_(nullptr) + , Parent_(nullptr) + , Height_(0) + , Tree_(nullptr) + { + } + inline ~TAvlTreeItem() noexcept { - Unlink(); - } - + Unlink(); + } + inline void Unlink() noexcept { - if (Tree_) { - Tree_->EraseImpl(this); - } - } - -private: - TAvlTreeItem* Left_; - TAvlTreeItem* Right_; - TAvlTreeItem* Parent_; - long Height_; - TTree* Tree_; -}; + if (Tree_) { + Tree_->EraseImpl(this); + } + } + +private: + TAvlTreeItem* Left_; + TAvlTreeItem* Right_; + TAvlTreeItem* Parent_; + long Height_; + TTree* Tree_; +}; diff --git a/library/cpp/containers/intrusive_avl_tree/ut/avltree_ut.cpp b/library/cpp/containers/intrusive_avl_tree/ut/avltree_ut.cpp index cab2365cce..256738c50a 100644 --- a/library/cpp/containers/intrusive_avl_tree/ut/avltree_ut.cpp +++ b/library/cpp/containers/intrusive_avl_tree/ut/avltree_ut.cpp @@ -3,29 +3,29 @@ #include <library/cpp/containers/intrusive_avl_tree/avltree.h> class TAvlTreeTest: public TTestBase { - UNIT_TEST_SUITE(TAvlTreeTest); - UNIT_TEST(TestLowerBound); + UNIT_TEST_SUITE(TAvlTreeTest); + UNIT_TEST(TestLowerBound); UNIT_TEST(TestIterator); - UNIT_TEST_SUITE_END(); + UNIT_TEST_SUITE_END(); -private: - void TestLowerBound(); +private: + void TestLowerBound(); void TestIterator(); - class TIt; - struct TItCompare { + class TIt; + struct TItCompare { static inline bool Compare(const TIt& l, const TIt& r) noexcept; - }; + }; - class TIt: public TAvlTreeItem<TIt, TItCompare> { - public: - TIt(int val = 0) - : Val(val) - { - } + class TIt: public TAvlTreeItem<TIt, TItCompare> { + public: + TIt(int val = 0) + : Val(val) + { + } - int Val; - }; + int Val; + }; using TIts = TAvlTree<TIt, TItCompare>; }; diff --git a/library/cpp/containers/intrusive_avl_tree/ut/ya.make b/library/cpp/containers/intrusive_avl_tree/ut/ya.make index 87920306d7..ef8fb95b6e 100644 --- a/library/cpp/containers/intrusive_avl_tree/ut/ya.make +++ b/library/cpp/containers/intrusive_avl_tree/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/containers/intrusive_avl_tree) - + OWNER( pg g:util ) - -SRCS( - avltree_ut.cpp -) - -END() + +SRCS( + avltree_ut.cpp +) + +END() diff --git a/library/cpp/containers/intrusive_avl_tree/ya.make b/library/cpp/containers/intrusive_avl_tree/ya.make index 6b061f2760..a8e3211340 100644 --- a/library/cpp/containers/intrusive_avl_tree/ya.make +++ b/library/cpp/containers/intrusive_avl_tree/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - +LIBRARY() + OWNER( pg g:util ) - -SRCS( - avltree.cpp -) - -END() + +SRCS( + avltree.cpp +) + +END() diff --git a/library/cpp/containers/intrusive_rb_tree/fuzz/rb_tree_fuzzing.cpp b/library/cpp/containers/intrusive_rb_tree/fuzz/rb_tree_fuzzing.cpp index 92370760b5..3bf84258c1 100644 --- a/library/cpp/containers/intrusive_rb_tree/fuzz/rb_tree_fuzzing.cpp +++ b/library/cpp/containers/intrusive_rb_tree/fuzz/rb_tree_fuzzing.cpp @@ -11,31 +11,31 @@ struct TCmp { } template <class T> - static inline bool Compare(const T& l, ui8 r) { + static inline bool Compare(const T& l, ui8 r) { return l.N < r; } template <class T> - static inline bool Compare(ui8 l, const T& r) { + static inline bool Compare(ui8 l, const T& r) { return l < r.N; } }; class TNode: public TRbTreeItem<TNode, TCmp> { public: - inline TNode(ui8 n) noexcept + inline TNode(ui8 n) noexcept : N(n) { } - ui8 N; + ui8 N; }; using TTree = TRbTree<TNode, TCmp>; -extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { TDeque<TNode> records; - const ui8 half = 128u; + const ui8 half = 128u; TTree tree; for (size_t i = 0; i < size; ++i) { if (data[i] / half == 0) { diff --git a/library/cpp/containers/intrusive_rb_tree/rb_tree.cpp b/library/cpp/containers/intrusive_rb_tree/rb_tree.cpp index 535b536c41..85b35e2634 100644 --- a/library/cpp/containers/intrusive_rb_tree/rb_tree.cpp +++ b/library/cpp/containers/intrusive_rb_tree/rb_tree.cpp @@ -1 +1 @@ -#include "rb_tree.h" +#include "rb_tree.h" diff --git a/library/cpp/containers/intrusive_rb_tree/rb_tree.h b/library/cpp/containers/intrusive_rb_tree/rb_tree.h index 0259452a14..ed94b1794d 100644 --- a/library/cpp/containers/intrusive_rb_tree/rb_tree.h +++ b/library/cpp/containers/intrusive_rb_tree/rb_tree.h @@ -1,48 +1,48 @@ #pragma once - -#include <util/generic/utility.h> -#include <util/generic/yexception.h> - + +#include <util/generic/utility.h> +#include <util/generic/yexception.h> + using TRbTreeColorType = bool; - -#define RBTreeRed false -#define RBTreeBlack true - -struct TRbTreeNodeBase { + +#define RBTreeRed false +#define RBTreeBlack true + +struct TRbTreeNodeBase { using TColorType = TRbTreeColorType; using TBasePtr = TRbTreeNodeBase*; - - TColorType Color_; - TBasePtr Parent_; - TBasePtr Left_; - TBasePtr Right_; + + TColorType Color_; + TBasePtr Parent_; + TBasePtr Left_; + TBasePtr Right_; size_t Children_; - + inline TRbTreeNodeBase() noexcept { - ReInitNode(); - } - + ReInitNode(); + } + inline void ReInitNode() noexcept { - Color_ = RBTreeBlack; + Color_ = RBTreeBlack; Parent_ = nullptr; Left_ = nullptr; Right_ = nullptr; Children_ = 1; - } - - static TBasePtr MinimumNode(TBasePtr x) { + } + + static TBasePtr MinimumNode(TBasePtr x) { while (x->Left_ != nullptr) - x = x->Left_; - - return x; - } - - static TBasePtr MaximumNode(TBasePtr x) { + x = x->Left_; + + return x; + } + + static TBasePtr MaximumNode(TBasePtr x) { while (x->Right_ != nullptr) - x = x->Right_; - - return x; - } + x = x->Right_; + + return x; + } static TBasePtr ByIndex(TBasePtr x, size_t index) { if (x->Left_ != nullptr) { @@ -56,300 +56,300 @@ struct TRbTreeNodeBase { ythrow yexception() << "index not found"; return ByIndex(x->Right_, index - 1); } -}; - -struct TRbTreeBaseIterator; - -template <class TDummy> -class TRbGlobal { -public: +}; + +struct TRbTreeBaseIterator; + +template <class TDummy> +class TRbGlobal { +public: using TBasePtr = TRbTreeNodeBase*; - - static void Rebalance(TBasePtr x, TBasePtr& root); - static TBasePtr RebalanceForErase(TBasePtr z, TBasePtr& root, TBasePtr& leftmost, TBasePtr& rightmost); + + static void Rebalance(TBasePtr x, TBasePtr& root); + static TBasePtr RebalanceForErase(TBasePtr z, TBasePtr& root, TBasePtr& leftmost, TBasePtr& rightmost); static void DecrementChildrenUntilRoot(TBasePtr x, TBasePtr root); static void RecalcChildren(TBasePtr x); - - static TBasePtr IncrementNode(TBasePtr); - static TBasePtr DecrementNode(TBasePtr); - static void RotateLeft(TBasePtr x, TBasePtr& root); - static void RotateRight(TBasePtr x, TBasePtr& root); -}; - + + static TBasePtr IncrementNode(TBasePtr); + static TBasePtr DecrementNode(TBasePtr); + static void RotateLeft(TBasePtr x, TBasePtr& root); + static void RotateRight(TBasePtr x, TBasePtr& root); +}; + using TRbGlobalInst = TRbGlobal<bool>; - -struct TRbTreeBaseIterator { + +struct TRbTreeBaseIterator { using TBasePtr = TRbTreeNodeBase*; - TBasePtr Node_; - + TBasePtr Node_; + inline TRbTreeBaseIterator(TBasePtr x = nullptr) noexcept - : Node_(x) - { - } -}; - -template <class TValue, class TTraits> -struct TRbTreeIterator: public TRbTreeBaseIterator { + : Node_(x) + { + } +}; + +template <class TValue, class TTraits> +struct TRbTreeIterator: public TRbTreeBaseIterator { using TReference = typename TTraits::TReference; using TPointer = typename TTraits::TPointer; using TSelf = TRbTreeIterator<TValue, TTraits>; using TBasePtr = TRbTreeNodeBase*; - + inline TRbTreeIterator() noexcept = default; - - template <class T1> + + template <class T1> inline TRbTreeIterator(const T1& x) noexcept - : TRbTreeBaseIterator(x) - { - } - + : TRbTreeBaseIterator(x) + { + } + inline TReference operator*() const noexcept { - return *static_cast<TValue*>(Node_); - } - + return *static_cast<TValue*>(Node_); + } + inline TPointer operator->() const noexcept { - return static_cast<TValue*>(Node_); - } - + return static_cast<TValue*>(Node_); + } + inline TSelf& operator++() noexcept { - Node_ = TRbGlobalInst::IncrementNode(Node_); - return *this; - } - + Node_ = TRbGlobalInst::IncrementNode(Node_); + return *this; + } + inline TSelf operator++(int) noexcept { - TSelf tmp = *this; - ++(*this); - return tmp; - } - + TSelf tmp = *this; + ++(*this); + return tmp; + } + inline TSelf& operator--() noexcept { - Node_ = TRbGlobalInst::DecrementNode(Node_); - return *this; - } - + Node_ = TRbGlobalInst::DecrementNode(Node_); + return *this; + } + inline TSelf operator--(int) noexcept { - TSelf tmp = *this; - --(*this); - return tmp; - } - - template <class T1> + TSelf tmp = *this; + --(*this); + return tmp; + } + + template <class T1> inline bool operator==(const T1& rhs) const noexcept { - return Node_ == rhs.Node_; - } - - template <class T1> + return Node_ == rhs.Node_; + } + + template <class T1> inline bool operator!=(const T1& rhs) const noexcept { - return Node_ != rhs.Node_; - } -}; - -template <class TValue, class TCmp> -class TRbTree { - struct TCmpAdaptor: public TCmp { + return Node_ != rhs.Node_; + } +}; + +template <class TValue, class TCmp> +class TRbTree { + struct TCmpAdaptor: public TCmp { inline TCmpAdaptor() noexcept = default; - + inline TCmpAdaptor(const TCmp& cmp) noexcept - : TCmp(cmp) - { - } - - template <class T1, class T2> - inline bool operator()(const T1& l, const T2& r) const { - return TCmp::Compare(l, r); - } - }; - - struct TNonConstTraits { + : TCmp(cmp) + { + } + + template <class T1, class T2> + inline bool operator()(const T1& l, const T2& r) const { + return TCmp::Compare(l, r); + } + }; + + struct TNonConstTraits { using TReference = TValue&; using TPointer = TValue*; - }; - - struct TConstTraits { + }; + + struct TConstTraits { using TReference = const TValue&; using TPointer = const TValue*; - }; - + }; + using TNodeBase = TRbTreeNodeBase; using TBasePtr = TRbTreeNodeBase*; using TColorType = TRbTreeColorType; - -public: - class TRealNode: public TNodeBase { - public: - inline TRealNode() + +public: + class TRealNode: public TNodeBase { + public: + inline TRealNode() : Tree_(nullptr) - { - } - + { + } + inline ~TRealNode() { - UnLink(); - } - + UnLink(); + } + inline void UnLink() noexcept { - if (Tree_) { - Tree_->EraseImpl(this); - ReInitNode(); + if (Tree_) { + Tree_->EraseImpl(this); + ReInitNode(); Tree_ = nullptr; - } - } - + } + } + inline void SetRbTreeParent(TRbTree* parent) noexcept { - Tree_ = parent; - } - + Tree_ = parent; + } + inline TRbTree* ParentTree() const noexcept { - return Tree_; - } - - private: - TRbTree* Tree_; - }; - + return Tree_; + } + + private: + TRbTree* Tree_; + }; + using TIterator = TRbTreeIterator<TValue, TNonConstTraits>; using TConstIterator = TRbTreeIterator<TValue, TConstTraits>; - + inline TRbTree() noexcept { - Init(); - } - + Init(); + } + inline TRbTree(const TCmp& cmp) noexcept - : KeyCompare_(cmp) - { - Init(); - } - + : KeyCompare_(cmp) + { + Init(); + } + inline void Init() noexcept { - Data_.Color_ = RBTreeRed; + Data_.Color_ = RBTreeRed; Data_.Parent_ = nullptr; - Data_.Left_ = &Data_; - Data_.Right_ = &Data_; - Data_.Children_ = 0; - } - - struct TDestroy { + Data_.Left_ = &Data_; + Data_.Right_ = &Data_; + Data_.Children_ = 0; + } + + struct TDestroy { inline void operator()(TValue& v) const noexcept { v.SetRbTreeParent(nullptr); - v.ReInitNode(); - } - }; - + v.ReInitNode(); + } + }; + inline ~TRbTree() { - ForEachNoOrder(TDestroy()); - } - + ForEachNoOrder(TDestroy()); + } + inline void Clear() noexcept { - ForEachNoOrder(TDestroy()); - Init(); - } - - template <class F> - inline void ForEachNoOrder(const F& f) { - ForEachNoOrder(Root(), f); - } - - template <class F> - inline void ForEachNoOrder(TNodeBase* n, const F& f) { - if (n && n != &Data_) { - ForEachNoOrder(n->Left_, f); - ForEachNoOrder(n->Right_, f); - f(ValueNode(n)); - } - } - + ForEachNoOrder(TDestroy()); + Init(); + } + + template <class F> + inline void ForEachNoOrder(const F& f) { + ForEachNoOrder(Root(), f); + } + + template <class F> + inline void ForEachNoOrder(TNodeBase* n, const F& f) { + if (n && n != &Data_) { + ForEachNoOrder(n->Left_, f); + ForEachNoOrder(n->Right_, f); + f(ValueNode(n)); + } + } + inline TIterator Begin() noexcept { - return LeftMost(); - } - + return LeftMost(); + } + inline TConstIterator Begin() const noexcept { - return LeftMost(); - } - + return LeftMost(); + } + inline TIterator End() noexcept { - return &this->Data_; - } - + return &this->Data_; + } + inline TConstIterator End() const noexcept { - return const_cast<TBasePtr>(&this->Data_); - } - + return const_cast<TBasePtr>(&this->Data_); + } + inline bool Empty() const noexcept { - return this->Begin() == this->End(); - } - + return this->Begin() == this->End(); + } + inline explicit operator bool() const noexcept { - return !this->Empty(); - } - - inline TIterator Insert(TValue* val) { - return Insert(*val); - } - - inline TIterator Insert(TValue& val) { - val.UnLink(); - - TBasePtr y = &this->Data_; - TBasePtr x = Root(); - + return !this->Empty(); + } + + inline TIterator Insert(TValue* val) { + return Insert(*val); + } + + inline TIterator Insert(TValue& val) { + val.UnLink(); + + TBasePtr y = &this->Data_; + TBasePtr x = Root(); + while (x != nullptr) { - ++(x->Children_); - y = x; - - if (KeyCompare_(ValueNode(&val), ValueNode(x))) { - x = LeftNode(x); - } else { - x = RightNode(x); - } - } - - return InsertImpl(y, &val, x); - } - - template <class F> - inline void ForEach(F& f) { - TIterator it = Begin(); - - while (it != End()) { - f(*it++); - } - } - + ++(x->Children_); + y = x; + + if (KeyCompare_(ValueNode(&val), ValueNode(x))) { + x = LeftNode(x); + } else { + x = RightNode(x); + } + } + + return InsertImpl(y, &val, x); + } + + template <class F> + inline void ForEach(F& f) { + TIterator it = Begin(); + + while (it != End()) { + f(*it++); + } + } + inline void Erase(TValue& val) noexcept { - val.UnLink(); - } - + val.UnLink(); + } + inline void Erase(TValue* val) noexcept { - Erase(*val); - } - + Erase(*val); + } + inline void Erase(TIterator pos) noexcept { - Erase(*pos); - } - + Erase(*pos); + } + inline void EraseImpl(TNodeBase* val) noexcept { - TRbGlobalInst::RebalanceForErase(val, this->Data_.Parent_, this->Data_.Left_, this->Data_.Right_); - } - - template <class T1> - inline TValue* Find(const T1& k) const { + TRbGlobalInst::RebalanceForErase(val, this->Data_.Parent_, this->Data_.Left_, this->Data_.Right_); + } + + template <class T1> + inline TValue* Find(const T1& k) const { TBasePtr y = nullptr; - TBasePtr x = Root(); // Current node. - + TBasePtr x = Root(); // Current node. + while (x != nullptr) - if (!KeyCompare_(ValueNode(x), k)) - y = x, x = LeftNode(x); - else - x = RightNode(x); - - if (y) { - if (KeyCompare_(k, ValueNode(y))) { + if (!KeyCompare_(ValueNode(x), k)) + y = x, x = LeftNode(x); + else + x = RightNode(x); + + if (y) { + if (KeyCompare_(k, ValueNode(y))) { y = nullptr; - } - } - - return static_cast<TValue*>(y); - } - + } + } + + return static_cast<TValue*>(y); + } + size_t GetIndex(TBasePtr x) const { size_t index = 0; @@ -370,33 +370,33 @@ public: return index; } - template <class T1> - inline TBasePtr LowerBound(const T1& k) const { - TBasePtr y = const_cast<TBasePtr>(&this->Data_); /* Last node which is not less than k. */ - TBasePtr x = Root(); /* Current node. */ - + template <class T1> + inline TBasePtr LowerBound(const T1& k) const { + TBasePtr y = const_cast<TBasePtr>(&this->Data_); /* Last node which is not less than k. */ + TBasePtr x = Root(); /* Current node. */ + while (x != nullptr) - if (!KeyCompare_(ValueNode(x), k)) - y = x, x = LeftNode(x); - else - x = RightNode(x); - - return y; - } - - template <class T1> - inline TBasePtr UpperBound(const T1& k) const { - TBasePtr y = const_cast<TBasePtr>(&this->Data_); /* Last node which is greater than k. */ - TBasePtr x = Root(); /* Current node. */ - + if (!KeyCompare_(ValueNode(x), k)) + y = x, x = LeftNode(x); + else + x = RightNode(x); + + return y; + } + + template <class T1> + inline TBasePtr UpperBound(const T1& k) const { + TBasePtr y = const_cast<TBasePtr>(&this->Data_); /* Last node which is greater than k. */ + TBasePtr x = Root(); /* Current node. */ + while (x != nullptr) - if (KeyCompare_(k, ValueNode(x))) - y = x, x = LeftNode(x); - else - x = RightNode(x); - - return y; - } + if (KeyCompare_(k, ValueNode(x))) + y = x, x = LeftNode(x); + else + x = RightNode(x); + + return y; + } template <class T1> inline size_t LessCount(const T1& k) const { @@ -432,184 +432,184 @@ public: return Root()->Children_ - GreaterCount<T1>(k); } - TValue* ByIndex(size_t index) { - return static_cast<TValue*>(TRbTreeNodeBase::ByIndex(Root(), index)); - } - -private: - // CRP 7/10/00 inserted argument on_right, which is another hint (meant to - // act like on_left and ignore a portion of the if conditions -- specify + TValue* ByIndex(size_t index) { + return static_cast<TValue*>(TRbTreeNodeBase::ByIndex(Root(), index)); + } + +private: + // CRP 7/10/00 inserted argument on_right, which is another hint (meant to + // act like on_left and ignore a portion of the if conditions -- specify // on_right != nullptr to bypass comparison as false or on_left != nullptr to bypass - // comparison as true) + // comparison as true) TIterator InsertImpl(TRbTreeNodeBase* parent, TRbTreeNodeBase* val, TRbTreeNodeBase* on_left = nullptr, TRbTreeNodeBase* on_right = nullptr) { - ValueNode(val).SetRbTreeParent(this); - TBasePtr new_node = val; - - if (parent == &this->Data_) { - LeftNode(parent) = new_node; - // also makes LeftMost() = new_node - Root() = new_node; - RightMost() = new_node; + ValueNode(val).SetRbTreeParent(this); + TBasePtr new_node = val; + + if (parent == &this->Data_) { + LeftNode(parent) = new_node; + // also makes LeftMost() = new_node + Root() = new_node; + RightMost() = new_node; } else if (on_right == nullptr && // If on_right != nullptr, the remainder fails to false (on_left != nullptr || // If on_left != nullptr, the remainder succeeds to true - KeyCompare_(ValueNode(val), ValueNode(parent)))) - { - LeftNode(parent) = new_node; - if (parent == LeftMost()) - // maintain LeftMost() pointing to min node - LeftMost() = new_node; - } else { - RightNode(parent) = new_node; - if (parent == RightMost()) - // maintain RightMost() pointing to max node - RightMost() = new_node; - } - ParentNode(new_node) = parent; - TRbGlobalInst::Rebalance(new_node, this->Data_.Parent_); - return new_node; - } - - TBasePtr Root() const { - return this->Data_.Parent_; - } - - TBasePtr LeftMost() const { - return this->Data_.Left_; - } - - TBasePtr RightMost() const { - return this->Data_.Right_; - } - - TBasePtr& Root() { - return this->Data_.Parent_; - } - - TBasePtr& LeftMost() { - return this->Data_.Left_; - } - - TBasePtr& RightMost() { - return this->Data_.Right_; - } - - static TBasePtr& LeftNode(TBasePtr x) { - return x->Left_; - } - - static TBasePtr& RightNode(TBasePtr x) { - return x->Right_; - } - - static TBasePtr& ParentNode(TBasePtr x) { - return x->Parent_; - } - - static TValue& ValueNode(TBasePtr x) { - return *static_cast<TValue*>(x); - } - - static TBasePtr MinimumNode(TBasePtr x) { - return TRbTreeNodeBase::MinimumNode(x); - } - - static TBasePtr MaximumNode(TBasePtr x) { - return TRbTreeNodeBase::MaximumNode(x); - } - -private: - TCmpAdaptor KeyCompare_; - TNodeBase Data_; -}; - -template <class TValue, class TCmp> -class TRbTreeItem: public TRbTree<TValue, TCmp>::TRealNode { -}; - -template <class TDummy> -void TRbGlobal<TDummy>::RotateLeft(TRbTreeNodeBase* x, TRbTreeNodeBase*& root) { - TRbTreeNodeBase* y = x->Right_; - x->Right_ = y->Left_; + KeyCompare_(ValueNode(val), ValueNode(parent)))) + { + LeftNode(parent) = new_node; + if (parent == LeftMost()) + // maintain LeftMost() pointing to min node + LeftMost() = new_node; + } else { + RightNode(parent) = new_node; + if (parent == RightMost()) + // maintain RightMost() pointing to max node + RightMost() = new_node; + } + ParentNode(new_node) = parent; + TRbGlobalInst::Rebalance(new_node, this->Data_.Parent_); + return new_node; + } + + TBasePtr Root() const { + return this->Data_.Parent_; + } + + TBasePtr LeftMost() const { + return this->Data_.Left_; + } + + TBasePtr RightMost() const { + return this->Data_.Right_; + } + + TBasePtr& Root() { + return this->Data_.Parent_; + } + + TBasePtr& LeftMost() { + return this->Data_.Left_; + } + + TBasePtr& RightMost() { + return this->Data_.Right_; + } + + static TBasePtr& LeftNode(TBasePtr x) { + return x->Left_; + } + + static TBasePtr& RightNode(TBasePtr x) { + return x->Right_; + } + + static TBasePtr& ParentNode(TBasePtr x) { + return x->Parent_; + } + + static TValue& ValueNode(TBasePtr x) { + return *static_cast<TValue*>(x); + } + + static TBasePtr MinimumNode(TBasePtr x) { + return TRbTreeNodeBase::MinimumNode(x); + } + + static TBasePtr MaximumNode(TBasePtr x) { + return TRbTreeNodeBase::MaximumNode(x); + } + +private: + TCmpAdaptor KeyCompare_; + TNodeBase Data_; +}; + +template <class TValue, class TCmp> +class TRbTreeItem: public TRbTree<TValue, TCmp>::TRealNode { +}; + +template <class TDummy> +void TRbGlobal<TDummy>::RotateLeft(TRbTreeNodeBase* x, TRbTreeNodeBase*& root) { + TRbTreeNodeBase* y = x->Right_; + x->Right_ = y->Left_; if (y->Left_ != nullptr) - y->Left_->Parent_ = x; - y->Parent_ = x->Parent_; - - if (x == root) - root = y; - else if (x == x->Parent_->Left_) - x->Parent_->Left_ = y; - else - x->Parent_->Right_ = y; - y->Left_ = x; - x->Parent_ = y; + y->Left_->Parent_ = x; + y->Parent_ = x->Parent_; + + if (x == root) + root = y; + else if (x == x->Parent_->Left_) + x->Parent_->Left_ = y; + else + x->Parent_->Right_ = y; + y->Left_ = x; + x->Parent_ = y; y->Children_ = x->Children_; x->Children_ = ((x->Left_) ? x->Left_->Children_ : 0) + ((x->Right_) ? x->Right_->Children_ : 0) + 1; -} - -template <class TDummy> -void TRbGlobal<TDummy>::RotateRight(TRbTreeNodeBase* x, TRbTreeNodeBase*& root) { - TRbTreeNodeBase* y = x->Left_; - x->Left_ = y->Right_; +} + +template <class TDummy> +void TRbGlobal<TDummy>::RotateRight(TRbTreeNodeBase* x, TRbTreeNodeBase*& root) { + TRbTreeNodeBase* y = x->Left_; + x->Left_ = y->Right_; if (y->Right_ != nullptr) - y->Right_->Parent_ = x; - y->Parent_ = x->Parent_; - - if (x == root) - root = y; - else if (x == x->Parent_->Right_) - x->Parent_->Right_ = y; - else - x->Parent_->Left_ = y; - y->Right_ = x; - x->Parent_ = y; + y->Right_->Parent_ = x; + y->Parent_ = x->Parent_; + + if (x == root) + root = y; + else if (x == x->Parent_->Right_) + x->Parent_->Right_ = y; + else + x->Parent_->Left_ = y; + y->Right_ = x; + x->Parent_ = y; y->Children_ = x->Children_; x->Children_ = ((x->Left_) ? x->Left_->Children_ : 0) + ((x->Right_) ? x->Right_->Children_ : 0) + 1; -} - -template <class TDummy> -void TRbGlobal<TDummy>::Rebalance(TRbTreeNodeBase* x, TRbTreeNodeBase*& root) { - x->Color_ = RBTreeRed; - while (x != root && x->Parent_->Color_ == RBTreeRed) { - if (x->Parent_ == x->Parent_->Parent_->Left_) { - TRbTreeNodeBase* y = x->Parent_->Parent_->Right_; - if (y && y->Color_ == RBTreeRed) { - x->Parent_->Color_ = RBTreeBlack; - y->Color_ = RBTreeBlack; - x->Parent_->Parent_->Color_ = RBTreeRed; - x = x->Parent_->Parent_; - } else { - if (x == x->Parent_->Right_) { - x = x->Parent_; - RotateLeft(x, root); - } - x->Parent_->Color_ = RBTreeBlack; - x->Parent_->Parent_->Color_ = RBTreeRed; - RotateRight(x->Parent_->Parent_, root); - } - } else { - TRbTreeNodeBase* y = x->Parent_->Parent_->Left_; - if (y && y->Color_ == RBTreeRed) { - x->Parent_->Color_ = RBTreeBlack; - y->Color_ = RBTreeBlack; - x->Parent_->Parent_->Color_ = RBTreeRed; - x = x->Parent_->Parent_; - } else { - if (x == x->Parent_->Left_) { - x = x->Parent_; - RotateRight(x, root); - } - x->Parent_->Color_ = RBTreeBlack; - x->Parent_->Parent_->Color_ = RBTreeRed; - RotateLeft(x->Parent_->Parent_, root); - } - } - } - root->Color_ = RBTreeBlack; -} - -template <class TDummy> +} + +template <class TDummy> +void TRbGlobal<TDummy>::Rebalance(TRbTreeNodeBase* x, TRbTreeNodeBase*& root) { + x->Color_ = RBTreeRed; + while (x != root && x->Parent_->Color_ == RBTreeRed) { + if (x->Parent_ == x->Parent_->Parent_->Left_) { + TRbTreeNodeBase* y = x->Parent_->Parent_->Right_; + if (y && y->Color_ == RBTreeRed) { + x->Parent_->Color_ = RBTreeBlack; + y->Color_ = RBTreeBlack; + x->Parent_->Parent_->Color_ = RBTreeRed; + x = x->Parent_->Parent_; + } else { + if (x == x->Parent_->Right_) { + x = x->Parent_; + RotateLeft(x, root); + } + x->Parent_->Color_ = RBTreeBlack; + x->Parent_->Parent_->Color_ = RBTreeRed; + RotateRight(x->Parent_->Parent_, root); + } + } else { + TRbTreeNodeBase* y = x->Parent_->Parent_->Left_; + if (y && y->Color_ == RBTreeRed) { + x->Parent_->Color_ = RBTreeBlack; + y->Color_ = RBTreeBlack; + x->Parent_->Parent_->Color_ = RBTreeRed; + x = x->Parent_->Parent_; + } else { + if (x == x->Parent_->Left_) { + x = x->Parent_; + RotateRight(x, root); + } + x->Parent_->Color_ = RBTreeBlack; + x->Parent_->Parent_->Color_ = RBTreeRed; + RotateLeft(x->Parent_->Parent_, root); + } + } + } + root->Color_ = RBTreeBlack; +} + +template <class TDummy> void TRbGlobal<TDummy>::RecalcChildren(TRbTreeNodeBase* x) { x->Children_ = ((x->Left_) ? x->Left_->Children_ : 0) + ((x->Right_) ? x->Right_->Children_ : 0) + 1; } @@ -625,46 +625,46 @@ void TRbGlobal<TDummy>::DecrementChildrenUntilRoot(TRbTreeNodeBase* x, TRbTreeNo } template <class TDummy> -TRbTreeNodeBase* TRbGlobal<TDummy>::RebalanceForErase(TRbTreeNodeBase* z, - TRbTreeNodeBase*& root, - TRbTreeNodeBase*& leftmost, - TRbTreeNodeBase*& rightmost) { - TRbTreeNodeBase* y = z; - TRbTreeNodeBase* x; - TRbTreeNodeBase* x_parent; - +TRbTreeNodeBase* TRbGlobal<TDummy>::RebalanceForErase(TRbTreeNodeBase* z, + TRbTreeNodeBase*& root, + TRbTreeNodeBase*& leftmost, + TRbTreeNodeBase*& rightmost) { + TRbTreeNodeBase* y = z; + TRbTreeNodeBase* x; + TRbTreeNodeBase* x_parent; + if (y->Left_ == nullptr) // z has at most one non-null child. y == z. - x = y->Right_; // x might be null. - else { - if (y->Right_ == nullptr) // z has exactly one non-null child. y == z. - x = y->Left_; // x is not null. - else { // z has two non-null children. Set y to - y = TRbTreeNodeBase::MinimumNode(y->Right_); // z's successor. x might be null. - x = y->Right_; - } - } - - if (y != z) { - // relink y in place of z. y is z's successor - z->Left_->Parent_ = y; - y->Left_ = z->Left_; - if (y != z->Right_) { - x_parent = y->Parent_; - if (x) - x->Parent_ = y->Parent_; - y->Parent_->Left_ = x; // y must be a child of mLeft - y->Right_ = z->Right_; - z->Right_->Parent_ = y; - } else - x_parent = y; - if (root == z) - root = y; - else if (z->Parent_->Left_ == z) - z->Parent_->Left_ = y; - else - z->Parent_->Right_ = y; - y->Parent_ = z->Parent_; - DoSwap(y->Color_, z->Color_); + x = y->Right_; // x might be null. + else { + if (y->Right_ == nullptr) // z has exactly one non-null child. y == z. + x = y->Left_; // x is not null. + else { // z has two non-null children. Set y to + y = TRbTreeNodeBase::MinimumNode(y->Right_); // z's successor. x might be null. + x = y->Right_; + } + } + + if (y != z) { + // relink y in place of z. y is z's successor + z->Left_->Parent_ = y; + y->Left_ = z->Left_; + if (y != z->Right_) { + x_parent = y->Parent_; + if (x) + x->Parent_ = y->Parent_; + y->Parent_->Left_ = x; // y must be a child of mLeft + y->Right_ = z->Right_; + z->Right_->Parent_ = y; + } else + x_parent = y; + if (root == z) + root = y; + else if (z->Parent_->Left_ == z) + z->Parent_->Left_ = y; + else + z->Parent_->Right_ = y; + y->Parent_ = z->Parent_; + DoSwap(y->Color_, z->Color_); RecalcChildren(y); if (x_parent != y) { @@ -673,146 +673,146 @@ TRbTreeNodeBase* TRbGlobal<TDummy>::RebalanceForErase(TRbTreeNodeBase* z, if (x_parent != root) { DecrementChildrenUntilRoot(x_parent->Parent_, root); } - y = z; - // y now points to node to be actually deleted - } else { - // y == z - x_parent = y->Parent_; - if (x) - x->Parent_ = y->Parent_; - if (root == z) - root = x; - else { - if (z->Parent_->Left_ == z) - z->Parent_->Left_ = x; - else - z->Parent_->Right_ = x; - DecrementChildrenUntilRoot(z->Parent_, root); // we lost y - } - - if (leftmost == z) { + y = z; + // y now points to node to be actually deleted + } else { + // y == z + x_parent = y->Parent_; + if (x) + x->Parent_ = y->Parent_; + if (root == z) + root = x; + else { + if (z->Parent_->Left_ == z) + z->Parent_->Left_ = x; + else + z->Parent_->Right_ = x; + DecrementChildrenUntilRoot(z->Parent_, root); // we lost y + } + + if (leftmost == z) { if (z->Right_ == nullptr) // z->mLeft must be null also - leftmost = z->Parent_; - // makes leftmost == _M_header if z == root - else - leftmost = TRbTreeNodeBase::MinimumNode(x); - } - if (rightmost == z) { + leftmost = z->Parent_; + // makes leftmost == _M_header if z == root + else + leftmost = TRbTreeNodeBase::MinimumNode(x); + } + if (rightmost == z) { if (z->Left_ == nullptr) // z->mRight must be null also - rightmost = z->Parent_; - // makes rightmost == _M_header if z == root - else // x == z->mLeft - rightmost = TRbTreeNodeBase::MaximumNode(x); - } - } - - if (y->Color_ != RBTreeRed) { + rightmost = z->Parent_; + // makes rightmost == _M_header if z == root + else // x == z->mLeft + rightmost = TRbTreeNodeBase::MaximumNode(x); + } + } + + if (y->Color_ != RBTreeRed) { while (x != root && (x == nullptr || x->Color_ == RBTreeBlack)) - if (x == x_parent->Left_) { - TRbTreeNodeBase* w = x_parent->Right_; - if (w->Color_ == RBTreeRed) { - w->Color_ = RBTreeBlack; - x_parent->Color_ = RBTreeRed; - RotateLeft(x_parent, root); - w = x_parent->Right_; - } + if (x == x_parent->Left_) { + TRbTreeNodeBase* w = x_parent->Right_; + if (w->Color_ == RBTreeRed) { + w->Color_ = RBTreeBlack; + x_parent->Color_ = RBTreeRed; + RotateLeft(x_parent, root); + w = x_parent->Right_; + } if ((w->Left_ == nullptr || - w->Left_->Color_ == RBTreeBlack) && + w->Left_->Color_ == RBTreeBlack) && (w->Right_ == nullptr || - w->Right_->Color_ == RBTreeBlack)) - { - w->Color_ = RBTreeRed; - x = x_parent; - x_parent = x_parent->Parent_; - } else { - if (w->Right_ == nullptr || w->Right_->Color_ == RBTreeBlack) { - if (w->Left_) - w->Left_->Color_ = RBTreeBlack; - w->Color_ = RBTreeRed; - RotateRight(w, root); - w = x_parent->Right_; - } - w->Color_ = x_parent->Color_; - x_parent->Color_ = RBTreeBlack; - if (w->Right_) - w->Right_->Color_ = RBTreeBlack; - RotateLeft(x_parent, root); - break; - } - } else { - // same as above, with mRight <-> mLeft. - TRbTreeNodeBase* w = x_parent->Left_; - if (w->Color_ == RBTreeRed) { - w->Color_ = RBTreeBlack; - x_parent->Color_ = RBTreeRed; - RotateRight(x_parent, root); - w = x_parent->Left_; - } + w->Right_->Color_ == RBTreeBlack)) + { + w->Color_ = RBTreeRed; + x = x_parent; + x_parent = x_parent->Parent_; + } else { + if (w->Right_ == nullptr || w->Right_->Color_ == RBTreeBlack) { + if (w->Left_) + w->Left_->Color_ = RBTreeBlack; + w->Color_ = RBTreeRed; + RotateRight(w, root); + w = x_parent->Right_; + } + w->Color_ = x_parent->Color_; + x_parent->Color_ = RBTreeBlack; + if (w->Right_) + w->Right_->Color_ = RBTreeBlack; + RotateLeft(x_parent, root); + break; + } + } else { + // same as above, with mRight <-> mLeft. + TRbTreeNodeBase* w = x_parent->Left_; + if (w->Color_ == RBTreeRed) { + w->Color_ = RBTreeBlack; + x_parent->Color_ = RBTreeRed; + RotateRight(x_parent, root); + w = x_parent->Left_; + } if ((w->Right_ == nullptr || - w->Right_->Color_ == RBTreeBlack) && + w->Right_->Color_ == RBTreeBlack) && (w->Left_ == nullptr || - w->Left_->Color_ == RBTreeBlack)) - { - w->Color_ = RBTreeRed; - x = x_parent; - x_parent = x_parent->Parent_; - } else { - if (w->Left_ == nullptr || w->Left_->Color_ == RBTreeBlack) { - if (w->Right_) - w->Right_->Color_ = RBTreeBlack; - w->Color_ = RBTreeRed; - RotateLeft(w, root); - w = x_parent->Left_; - } - w->Color_ = x_parent->Color_; - x_parent->Color_ = RBTreeBlack; - if (w->Left_) - w->Left_->Color_ = RBTreeBlack; - RotateRight(x_parent, root); - break; - } - } - if (x) - x->Color_ = RBTreeBlack; - } - return y; -} - -template <class TDummy> -TRbTreeNodeBase* TRbGlobal<TDummy>::DecrementNode(TRbTreeNodeBase* Node_) { - if (Node_->Color_ == RBTreeRed && Node_->Parent_->Parent_ == Node_) - Node_ = Node_->Right_; + w->Left_->Color_ == RBTreeBlack)) + { + w->Color_ = RBTreeRed; + x = x_parent; + x_parent = x_parent->Parent_; + } else { + if (w->Left_ == nullptr || w->Left_->Color_ == RBTreeBlack) { + if (w->Right_) + w->Right_->Color_ = RBTreeBlack; + w->Color_ = RBTreeRed; + RotateLeft(w, root); + w = x_parent->Left_; + } + w->Color_ = x_parent->Color_; + x_parent->Color_ = RBTreeBlack; + if (w->Left_) + w->Left_->Color_ = RBTreeBlack; + RotateRight(x_parent, root); + break; + } + } + if (x) + x->Color_ = RBTreeBlack; + } + return y; +} + +template <class TDummy> +TRbTreeNodeBase* TRbGlobal<TDummy>::DecrementNode(TRbTreeNodeBase* Node_) { + if (Node_->Color_ == RBTreeRed && Node_->Parent_->Parent_ == Node_) + Node_ = Node_->Right_; else if (Node_->Left_ != nullptr) { - Node_ = TRbTreeNodeBase::MaximumNode(Node_->Left_); - } else { - TBasePtr y = Node_->Parent_; - while (Node_ == y->Left_) { - Node_ = y; - y = y->Parent_; - } - Node_ = y; - } - return Node_; -} - -template <class TDummy> -TRbTreeNodeBase* TRbGlobal<TDummy>::IncrementNode(TRbTreeNodeBase* Node_) { + Node_ = TRbTreeNodeBase::MaximumNode(Node_->Left_); + } else { + TBasePtr y = Node_->Parent_; + while (Node_ == y->Left_) { + Node_ = y; + y = y->Parent_; + } + Node_ = y; + } + return Node_; +} + +template <class TDummy> +TRbTreeNodeBase* TRbGlobal<TDummy>::IncrementNode(TRbTreeNodeBase* Node_) { if (Node_->Right_ != nullptr) { - Node_ = TRbTreeNodeBase::MinimumNode(Node_->Right_); - } else { - TBasePtr y = Node_->Parent_; - while (Node_ == y->Right_) { - Node_ = y; - y = y->Parent_; - } - // check special case: This is necessary if mNode is the - // _M_head and the tree contains only a single node y. In - // that case parent, left and right all point to y! - if (Node_->Right_ != y) - Node_ = y; - } - return Node_; -} - -#undef RBTreeRed -#undef RBTreeBlack + Node_ = TRbTreeNodeBase::MinimumNode(Node_->Right_); + } else { + TBasePtr y = Node_->Parent_; + while (Node_ == y->Right_) { + Node_ = y; + y = y->Parent_; + } + // check special case: This is necessary if mNode is the + // _M_head and the tree contains only a single node y. In + // that case parent, left and right all point to y! + if (Node_->Right_ != y) + Node_ = y; + } + return Node_; +} + +#undef RBTreeRed +#undef RBTreeBlack diff --git a/library/cpp/containers/intrusive_rb_tree/rb_tree_ut.cpp b/library/cpp/containers/intrusive_rb_tree/rb_tree_ut.cpp index c34ed1fd9b..cea3ab4421 100644 --- a/library/cpp/containers/intrusive_rb_tree/rb_tree_ut.cpp +++ b/library/cpp/containers/intrusive_rb_tree/rb_tree_ut.cpp @@ -1,82 +1,82 @@ -#include "rb_tree.h" - +#include "rb_tree.h" + #include <library/cpp/testing/unittest/registar.h> - -#include <util/random/fast.h> -#include <util/random/easy.h> + +#include <util/random/fast.h> +#include <util/random/easy.h> #include <util/random/shuffle.h> - -class TRedBlackTreeTest: public TTestBase { - struct TCmp { - template <class T> - static inline bool Compare(const T& l, const T& r) { - return l.N < r.N; - } - - template <class T> - static inline bool Compare(const T& l, int r) { - return l.N < r; - } - - template <class T> - static inline bool Compare(int l, const T& r) { - return l < r.N; - } - }; - - class TNode: public TRbTreeItem<TNode, TCmp> { - public: + +class TRedBlackTreeTest: public TTestBase { + struct TCmp { + template <class T> + static inline bool Compare(const T& l, const T& r) { + return l.N < r.N; + } + + template <class T> + static inline bool Compare(const T& l, int r) { + return l.N < r; + } + + template <class T> + static inline bool Compare(int l, const T& r) { + return l < r.N; + } + }; + + class TNode: public TRbTreeItem<TNode, TCmp> { + public: inline TNode(int n) noexcept - : N(n) - { - } - - int N; - }; - + : N(n) + { + } + + int N; + }; + using TTree = TRbTree<TNode, TCmp>; - - UNIT_TEST_SUITE(TRedBlackTreeTest); - UNIT_TEST(TestEmpty) - UNIT_TEST(TestInsert) - UNIT_TEST(TestErase) - UNIT_TEST(TestFind) - UNIT_TEST(TestStress) + + UNIT_TEST_SUITE(TRedBlackTreeTest); + UNIT_TEST(TestEmpty) + UNIT_TEST(TestInsert) + UNIT_TEST(TestErase) + UNIT_TEST(TestFind) + UNIT_TEST(TestStress) UNIT_TEST(TestGettingIndexWithDifferentValues) UNIT_TEST(TestCheckChildrenAfterErase) UNIT_TEST(TestGettingIndexWithDifferentValuesAfterErase) UNIT_TEST(TestGettingIndexWithEqualValues) UNIT_TEST(TestLessCountOnEmptyTree) - UNIT_TEST_SUITE_END(); - -private: - inline void TestStress() { + UNIT_TEST_SUITE_END(); + +private: + inline void TestStress() { TVector<TSimpleSharedPtr<TNode>> nodes; - - for (int i = 0; i < 1000; ++i) { - nodes.push_back(new TNode(i)); - } - - TTree tree; - TReallyFastRng32 rnd(Random()); - - for (size_t i = 0; i < 1000000; ++i) { - tree.Insert(nodes[rnd.Uniform(nodes.size())].Get()); - } - - for (TTree::TConstIterator it = tree.Begin(); it != tree.End();) { - const int v1 = it->N; - - if (++it == tree.End()) { - break; - } - - const int v2 = it->N; - - UNIT_ASSERT(v1 < v2); - } - } - + + for (int i = 0; i < 1000; ++i) { + nodes.push_back(new TNode(i)); + } + + TTree tree; + TReallyFastRng32 rnd(Random()); + + for (size_t i = 0; i < 1000000; ++i) { + tree.Insert(nodes[rnd.Uniform(nodes.size())].Get()); + } + + for (TTree::TConstIterator it = tree.Begin(); it != tree.End();) { + const int v1 = it->N; + + if (++it == tree.End()) { + break; + } + + const int v2 = it->N; + + UNIT_ASSERT(v1 < v2); + } + } + inline void TestGettingIndexWithDifferentValues() { TVector<TSimpleSharedPtr<TNode>> nodes; size_t N = 1000; @@ -202,97 +202,97 @@ private: } } - inline void TestFind() { - TTree tree; - - { - TNode n1(1); - TNode n2(2); - TNode n3(3); - - tree.Insert(n1); - tree.Insert(n2); - tree.Insert(n3); - - UNIT_ASSERT_EQUAL(tree.Find(1)->N, 1); - UNIT_ASSERT_EQUAL(tree.Find(2)->N, 2); - UNIT_ASSERT_EQUAL(tree.Find(3)->N, 3); - - UNIT_ASSERT(!tree.Find(0)); - UNIT_ASSERT(!tree.Find(4)); - UNIT_ASSERT(!tree.Find(1234567)); - } - - UNIT_ASSERT(tree.Empty()); - } - - inline void TestEmpty() { - TTree tree; - - UNIT_ASSERT(tree.Empty()); - UNIT_ASSERT_EQUAL(tree.Begin(), tree.End()); - } - - inline void TestInsert() { - TTree tree; - - { - TNode n1(1); - TNode n2(2); - TNode n3(3); - - tree.Insert(n1); - tree.Insert(n2); - tree.Insert(n3); - - TTree::TConstIterator it = tree.Begin(); - - UNIT_ASSERT_EQUAL((it++)->N, 1); - UNIT_ASSERT_EQUAL((it++)->N, 2); - UNIT_ASSERT_EQUAL((it++)->N, 3); - UNIT_ASSERT_EQUAL(it, tree.End()); - } - - UNIT_ASSERT(tree.Empty()); - } - - inline void TestErase() { - TTree tree; - - { - TNode n1(1); - TNode n2(2); - TNode n3(3); - - tree.Insert(n1); - tree.Insert(n2); - tree.Insert(n3); - - TTree::TIterator it = tree.Begin(); - - tree.Erase(it++); - - UNIT_ASSERT_EQUAL(it, tree.Begin()); - UNIT_ASSERT_EQUAL(it->N, 2); - - tree.Erase(it++); - - UNIT_ASSERT_EQUAL(it, tree.Begin()); - UNIT_ASSERT_EQUAL(it->N, 3); - - tree.Erase(it++); - - UNIT_ASSERT_EQUAL(it, tree.Begin()); - UNIT_ASSERT_EQUAL(it, tree.End()); - } - - UNIT_ASSERT(tree.Empty()); - } + inline void TestFind() { + TTree tree; + + { + TNode n1(1); + TNode n2(2); + TNode n3(3); + + tree.Insert(n1); + tree.Insert(n2); + tree.Insert(n3); + + UNIT_ASSERT_EQUAL(tree.Find(1)->N, 1); + UNIT_ASSERT_EQUAL(tree.Find(2)->N, 2); + UNIT_ASSERT_EQUAL(tree.Find(3)->N, 3); + + UNIT_ASSERT(!tree.Find(0)); + UNIT_ASSERT(!tree.Find(4)); + UNIT_ASSERT(!tree.Find(1234567)); + } + + UNIT_ASSERT(tree.Empty()); + } + + inline void TestEmpty() { + TTree tree; + + UNIT_ASSERT(tree.Empty()); + UNIT_ASSERT_EQUAL(tree.Begin(), tree.End()); + } + + inline void TestInsert() { + TTree tree; + + { + TNode n1(1); + TNode n2(2); + TNode n3(3); + + tree.Insert(n1); + tree.Insert(n2); + tree.Insert(n3); + + TTree::TConstIterator it = tree.Begin(); + + UNIT_ASSERT_EQUAL((it++)->N, 1); + UNIT_ASSERT_EQUAL((it++)->N, 2); + UNIT_ASSERT_EQUAL((it++)->N, 3); + UNIT_ASSERT_EQUAL(it, tree.End()); + } + + UNIT_ASSERT(tree.Empty()); + } + + inline void TestErase() { + TTree tree; + + { + TNode n1(1); + TNode n2(2); + TNode n3(3); + + tree.Insert(n1); + tree.Insert(n2); + tree.Insert(n3); + + TTree::TIterator it = tree.Begin(); + + tree.Erase(it++); + + UNIT_ASSERT_EQUAL(it, tree.Begin()); + UNIT_ASSERT_EQUAL(it->N, 2); + + tree.Erase(it++); + + UNIT_ASSERT_EQUAL(it, tree.Begin()); + UNIT_ASSERT_EQUAL(it->N, 3); + + tree.Erase(it++); + + UNIT_ASSERT_EQUAL(it, tree.Begin()); + UNIT_ASSERT_EQUAL(it, tree.End()); + } + + UNIT_ASSERT(tree.Empty()); + } inline void TestLessCountOnEmptyTree() { TTree tree; UNIT_ASSERT_VALUES_EQUAL(0, tree.LessCount(TNode(1))); } -}; - -UNIT_TEST_SUITE_REGISTRATION(TRedBlackTreeTest); +}; + +UNIT_TEST_SUITE_REGISTRATION(TRedBlackTreeTest); diff --git a/library/cpp/containers/intrusive_rb_tree/ut/ya.make b/library/cpp/containers/intrusive_rb_tree/ut/ya.make index 6f1e3b38ee..15c74020dc 100644 --- a/library/cpp/containers/intrusive_rb_tree/ut/ya.make +++ b/library/cpp/containers/intrusive_rb_tree/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/containers/intrusive_rb_tree) - + OWNER( pg g:util ) - -SRCS( - rb_tree_ut.cpp -) - -END() + +SRCS( + rb_tree_ut.cpp +) + +END() diff --git a/library/cpp/containers/intrusive_rb_tree/ya.make b/library/cpp/containers/intrusive_rb_tree/ya.make index 2e5eddcfbe..ff2c0b5adf 100644 --- a/library/cpp/containers/intrusive_rb_tree/ya.make +++ b/library/cpp/containers/intrusive_rb_tree/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - +LIBRARY() + OWNER( pg g:util ) - -SRCS( - rb_tree.cpp -) - -END() + +SRCS( + rb_tree.cpp +) + +END() diff --git a/library/cpp/containers/paged_vector/paged_vector.cpp b/library/cpp/containers/paged_vector/paged_vector.cpp index e354caf09d..52f91880bf 100644 --- a/library/cpp/containers/paged_vector/paged_vector.cpp +++ b/library/cpp/containers/paged_vector/paged_vector.cpp @@ -1 +1 @@ -#include "paged_vector.h" +#include "paged_vector.h" diff --git a/library/cpp/containers/paged_vector/paged_vector.h b/library/cpp/containers/paged_vector/paged_vector.h index 6a3657d3ea..dbd1486a71 100644 --- a/library/cpp/containers/paged_vector/paged_vector.h +++ b/library/cpp/containers/paged_vector/paged_vector.h @@ -4,429 +4,429 @@ #include <util/generic/vector.h> #include <util/generic/yexception.h> -#include <iterator> - +#include <iterator> + namespace NPagedVector { - template <class T, ui32 PageSize = 1u << 20, class A = std::allocator<T>> - class TPagedVector; - - namespace NPrivate { - template <class T, class TT, ui32 PageSize, class A> - struct TPagedVectorIterator { - private: - friend class TPagedVector<TT, PageSize, A>; - typedef TPagedVector<TT, PageSize, A> TVec; - typedef TPagedVectorIterator<T, TT, PageSize, A> TSelf; - size_t Offset; - TVec* Vector; - - template <class T1, class TT1, ui32 PageSize1, class A1> - friend struct TPagedVectorIterator; - - public: - TPagedVectorIterator() - : Offset() - , Vector() - { - } - - TPagedVectorIterator(TVec* vector, size_t offset) - : Offset(offset) - , Vector(vector) - { - } - - template <class T1, class TT1, ui32 PageSize1, class A1> - TPagedVectorIterator(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) - : Offset(it.Offset) - , Vector(it.Vector) - { - } - - T& operator*() const { - return (*Vector)[Offset]; - } - - T* operator->() const { - return &(**this); - } - - template <class T1, class TT1, ui32 PageSize1, class A1> - bool operator==(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { - return Offset == it.Offset; - } - - template <class T1, class TT1, ui32 PageSize1, class A1> - bool operator!=(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { - return !(*this == it); - } - - template <class T1, class TT1, ui32 PageSize1, class A1> - bool operator<(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { - return Offset < it.Offset; - } - - template <class T1, class TT1, ui32 PageSize1, class A1> - bool operator<=(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { - return Offset <= it.Offset; - } - - template <class T1, class TT1, ui32 PageSize1, class A1> - bool operator>(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { - return !(*this <= it); - } - - template <class T1, class TT1, ui32 PageSize1, class A1> - bool operator>=(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { - return !(*this < it); - } - - template <class T1, class TT1, ui32 PageSize1, class A1> - ptrdiff_t operator-(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { - return Offset - it.Offset; - } - - TSelf& operator+=(ptrdiff_t off) { - Offset += off; - return *this; - } - - TSelf& operator-=(ptrdiff_t off) { - return this->operator+=(-off); - } - - TSelf& operator++() { - return this->operator+=(1); - } - - TSelf& operator--() { - return this->operator+=(-1); - } - - TSelf operator++(int) { - TSelf it = *this; - this->operator+=(1); - return it; - } - - TSelf operator--(int) { - TSelf it = *this; - this->operator+=(-1); - return it; - } - - TSelf operator+(ptrdiff_t off) { - TSelf res = *this; - res += off; - return res; - } - - TSelf operator-(ptrdiff_t off) { - return this->operator+(-off); - } - - size_t GetOffset() { - return Offset; - } - }; + template <class T, ui32 PageSize = 1u << 20, class A = std::allocator<T>> + class TPagedVector; + + namespace NPrivate { + template <class T, class TT, ui32 PageSize, class A> + struct TPagedVectorIterator { + private: + friend class TPagedVector<TT, PageSize, A>; + typedef TPagedVector<TT, PageSize, A> TVec; + typedef TPagedVectorIterator<T, TT, PageSize, A> TSelf; + size_t Offset; + TVec* Vector; + + template <class T1, class TT1, ui32 PageSize1, class A1> + friend struct TPagedVectorIterator; + + public: + TPagedVectorIterator() + : Offset() + , Vector() + { + } + + TPagedVectorIterator(TVec* vector, size_t offset) + : Offset(offset) + , Vector(vector) + { + } + + template <class T1, class TT1, ui32 PageSize1, class A1> + TPagedVectorIterator(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) + : Offset(it.Offset) + , Vector(it.Vector) + { + } + + T& operator*() const { + return (*Vector)[Offset]; + } + + T* operator->() const { + return &(**this); + } + + template <class T1, class TT1, ui32 PageSize1, class A1> + bool operator==(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { + return Offset == it.Offset; + } + + template <class T1, class TT1, ui32 PageSize1, class A1> + bool operator!=(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { + return !(*this == it); + } + + template <class T1, class TT1, ui32 PageSize1, class A1> + bool operator<(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { + return Offset < it.Offset; + } + + template <class T1, class TT1, ui32 PageSize1, class A1> + bool operator<=(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { + return Offset <= it.Offset; + } + + template <class T1, class TT1, ui32 PageSize1, class A1> + bool operator>(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { + return !(*this <= it); + } + + template <class T1, class TT1, ui32 PageSize1, class A1> + bool operator>=(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { + return !(*this < it); + } + + template <class T1, class TT1, ui32 PageSize1, class A1> + ptrdiff_t operator-(const TPagedVectorIterator<T1, TT1, PageSize1, A1>& it) const { + return Offset - it.Offset; + } + + TSelf& operator+=(ptrdiff_t off) { + Offset += off; + return *this; + } + + TSelf& operator-=(ptrdiff_t off) { + return this->operator+=(-off); + } + + TSelf& operator++() { + return this->operator+=(1); + } + + TSelf& operator--() { + return this->operator+=(-1); + } + + TSelf operator++(int) { + TSelf it = *this; + this->operator+=(1); + return it; + } + + TSelf operator--(int) { + TSelf it = *this; + this->operator+=(-1); + return it; + } + + TSelf operator+(ptrdiff_t off) { + TSelf res = *this; + res += off; + return res; + } + + TSelf operator-(ptrdiff_t off) { + return this->operator+(-off); + } + + size_t GetOffset() { + return Offset; + } + }; } } -namespace std { - template <class T, class TT, ui32 PageSize, class A> - struct iterator_traits<NPagedVector::NPrivate::TPagedVectorIterator<T, TT, PageSize, A>> { - typedef ptrdiff_t difference_type; - typedef T value_type; - typedef T* pointer; - typedef T& reference; - typedef random_access_iterator_tag iterator_category; - }; +namespace std { + template <class T, class TT, ui32 PageSize, class A> + struct iterator_traits<NPagedVector::NPrivate::TPagedVectorIterator<T, TT, PageSize, A>> { + typedef ptrdiff_t difference_type; + typedef T value_type; + typedef T* pointer; + typedef T& reference; + typedef random_access_iterator_tag iterator_category; + }; } namespace NPagedVector { - //2-level radix tree - template <class T, ui32 PageSize, class A> - class TPagedVector: private TVector<TSimpleSharedPtr<TVector<T, A>>, A> { - static_assert(PageSize, "expect PageSize"); - - typedef TVector<T, A> TPage; - typedef TVector<TSimpleSharedPtr<TPage>, A> TPages; - typedef TPagedVector<T, PageSize, A> TSelf; - - public: - typedef NPrivate::TPagedVectorIterator<T, T, PageSize, A> iterator; - typedef NPrivate::TPagedVectorIterator<const T, T, PageSize, A> const_iterator; - typedef std::reverse_iterator<iterator> reverse_iterator; - typedef std::reverse_iterator<const_iterator> const_reverse_iterator; - typedef T value_type; - typedef value_type& reference; - typedef const value_type& const_reference; - - TPagedVector() = default; - - template <typename TIter> - TPagedVector(TIter b, TIter e) { - append(b, e); - } - - iterator begin() { - return iterator(this, 0); - } - - const_iterator begin() const { - return const_iterator((TSelf*)this, 0); - } - - iterator end() { - return iterator(this, size()); - } - - const_iterator end() const { - return const_iterator((TSelf*)this, size()); - } - - reverse_iterator rbegin() { - return reverse_iterator(end()); - } - - const_reverse_iterator rbegin() const { - return const_reverse_iterator(end()); - } - - reverse_iterator rend() { - return reverse_iterator(begin()); - } - - const_reverse_iterator rend() const { - return const_reverse_iterator(begin()); - } - - void swap(TSelf& v) { - TPages::swap((TPages&)v); - } - - private: - static size_t PageNumber(size_t idx) { - return idx / PageSize; - } - - static size_t InPageIndex(size_t idx) { - return idx % PageSize; - } - - static size_t Index(size_t pnum, size_t poff) { - return pnum * PageSize + poff; - } - - TPage& PageAt(size_t pnum) const { - return *TPages::at(pnum); - } - - TPage& CurrentPage() const { - return *TPages::back(); - } - - size_t CurrentPageSize() const { - return TPages::empty() ? 0 : CurrentPage().size(); - } - - size_t NPages() const { - return TPages::size(); - } - - void AllocateNewPage() { - TPages::push_back(new TPage()); - CurrentPage().reserve(PageSize); - } - - void MakeNewPage() { + //2-level radix tree + template <class T, ui32 PageSize, class A> + class TPagedVector: private TVector<TSimpleSharedPtr<TVector<T, A>>, A> { + static_assert(PageSize, "expect PageSize"); + + typedef TVector<T, A> TPage; + typedef TVector<TSimpleSharedPtr<TPage>, A> TPages; + typedef TPagedVector<T, PageSize, A> TSelf; + + public: + typedef NPrivate::TPagedVectorIterator<T, T, PageSize, A> iterator; + typedef NPrivate::TPagedVectorIterator<const T, T, PageSize, A> const_iterator; + typedef std::reverse_iterator<iterator> reverse_iterator; + typedef std::reverse_iterator<const_iterator> const_reverse_iterator; + typedef T value_type; + typedef value_type& reference; + typedef const value_type& const_reference; + + TPagedVector() = default; + + template <typename TIter> + TPagedVector(TIter b, TIter e) { + append(b, e); + } + + iterator begin() { + return iterator(this, 0); + } + + const_iterator begin() const { + return const_iterator((TSelf*)this, 0); + } + + iterator end() { + return iterator(this, size()); + } + + const_iterator end() const { + return const_iterator((TSelf*)this, size()); + } + + reverse_iterator rbegin() { + return reverse_iterator(end()); + } + + const_reverse_iterator rbegin() const { + return const_reverse_iterator(end()); + } + + reverse_iterator rend() { + return reverse_iterator(begin()); + } + + const_reverse_iterator rend() const { + return const_reverse_iterator(begin()); + } + + void swap(TSelf& v) { + TPages::swap((TPages&)v); + } + + private: + static size_t PageNumber(size_t idx) { + return idx / PageSize; + } + + static size_t InPageIndex(size_t idx) { + return idx % PageSize; + } + + static size_t Index(size_t pnum, size_t poff) { + return pnum * PageSize + poff; + } + + TPage& PageAt(size_t pnum) const { + return *TPages::at(pnum); + } + + TPage& CurrentPage() const { + return *TPages::back(); + } + + size_t CurrentPageSize() const { + return TPages::empty() ? 0 : CurrentPage().size(); + } + + size_t NPages() const { + return TPages::size(); + } + + void AllocateNewPage() { + TPages::push_back(new TPage()); + CurrentPage().reserve(PageSize); + } + + void MakeNewPage() { AllocateNewPage(); - CurrentPage().resize(PageSize); - } - - void PrepareAppend() { - if (TPages::empty() || CurrentPage().size() + 1 > PageSize) - AllocateNewPage(); - } - - public: - size_t size() const { - return empty() ? 0 : (NPages() - 1) * PageSize + CurrentPage().size(); - } - - bool empty() const { - return TPages::empty() || 1 == NPages() && CurrentPage().empty(); - } - - explicit operator bool() const noexcept { - return !empty(); - } - - void emplace_back() { - PrepareAppend(); - CurrentPage().emplace_back(); - } - - void push_back(const_reference t) { - PrepareAppend(); - CurrentPage().push_back(t); - } - - void pop_back() { - if (CurrentPage().empty()) - TPages::pop_back(); - CurrentPage().pop_back(); - } - - template <typename TIter> - void append(TIter b, TIter e) { - size_t sz = e - b; - size_t sz1 = Min<size_t>(sz, PageSize - CurrentPageSize()); - size_t sz2 = (sz - sz1) / PageSize; - size_t sz3 = (sz - sz1) % PageSize; - - if (sz1) { - PrepareAppend(); - TPage& p = CurrentPage(); - p.insert(p.end(), b, b + sz1); - } - - for (size_t i = 0; i < sz2; ++i) { - AllocateNewPage(); - TPage& p = CurrentPage(); - p.insert(p.end(), b + sz1 + i * PageSize, b + sz1 + (i + 1) * PageSize); - } - - if (sz3) { - AllocateNewPage(); - TPage& p = CurrentPage(); - p.insert(p.end(), b + sz1 + sz2 * PageSize, e); - } - } - - iterator erase(iterator it) { - size_t pnum = PageNumber(it.Offset); - size_t pidx = InPageIndex(it.Offset); - - if (CurrentPage().empty()) - TPages::pop_back(); - - for (size_t p = NPages() - 1; p > pnum; --p) { - PageAt(p - 1).push_back(PageAt(p).front()); - PageAt(p).erase(PageAt(p).begin()); - } - - PageAt(pnum).erase(PageAt(pnum).begin() + pidx); - return it; - } - - iterator erase(iterator b, iterator e) { - // todo : suboptimal! - while (b != e) { - b = erase(b); - --e; - } - - return b; - } - - iterator insert(iterator it, const value_type& v) { - size_t pnum = PageNumber(it.Offset); - size_t pidx = InPageIndex(it.Offset); - + CurrentPage().resize(PageSize); + } + + void PrepareAppend() { + if (TPages::empty() || CurrentPage().size() + 1 > PageSize) + AllocateNewPage(); + } + + public: + size_t size() const { + return empty() ? 0 : (NPages() - 1) * PageSize + CurrentPage().size(); + } + + bool empty() const { + return TPages::empty() || 1 == NPages() && CurrentPage().empty(); + } + + explicit operator bool() const noexcept { + return !empty(); + } + + void emplace_back() { + PrepareAppend(); + CurrentPage().emplace_back(); + } + + void push_back(const_reference t) { PrepareAppend(); + CurrentPage().push_back(t); + } + + void pop_back() { + if (CurrentPage().empty()) + TPages::pop_back(); + CurrentPage().pop_back(); + } + + template <typename TIter> + void append(TIter b, TIter e) { + size_t sz = e - b; + size_t sz1 = Min<size_t>(sz, PageSize - CurrentPageSize()); + size_t sz2 = (sz - sz1) / PageSize; + size_t sz3 = (sz - sz1) % PageSize; + + if (sz1) { + PrepareAppend(); + TPage& p = CurrentPage(); + p.insert(p.end(), b, b + sz1); + } + + for (size_t i = 0; i < sz2; ++i) { + AllocateNewPage(); + TPage& p = CurrentPage(); + p.insert(p.end(), b + sz1 + i * PageSize, b + sz1 + (i + 1) * PageSize); + } + + if (sz3) { + AllocateNewPage(); + TPage& p = CurrentPage(); + p.insert(p.end(), b + sz1 + sz2 * PageSize, e); + } + } + + iterator erase(iterator it) { + size_t pnum = PageNumber(it.Offset); + size_t pidx = InPageIndex(it.Offset); + + if (CurrentPage().empty()) + TPages::pop_back(); + + for (size_t p = NPages() - 1; p > pnum; --p) { + PageAt(p - 1).push_back(PageAt(p).front()); + PageAt(p).erase(PageAt(p).begin()); + } + + PageAt(pnum).erase(PageAt(pnum).begin() + pidx); + return it; + } + + iterator erase(iterator b, iterator e) { + // todo : suboptimal! + while (b != e) { + b = erase(b); + --e; + } + + return b; + } + + iterator insert(iterator it, const value_type& v) { + size_t pnum = PageNumber(it.Offset); + size_t pidx = InPageIndex(it.Offset); + + PrepareAppend(); + + for (size_t p = NPages() - 1; p > pnum; --p) { + PageAt(p).insert(PageAt(p).begin(), PageAt(p - 1).back()); + PageAt(p - 1).pop_back(); + } + + PageAt(pnum).insert(PageAt(pnum).begin() + pidx, v); + return it; + } + + template <typename TIter> + void insert(iterator it, TIter b, TIter e) { + // todo : suboptimal! + for (; b != e; ++b, ++it) + it = insert(it, *b); + } + + reference front() { + return TPages::front()->front(); + } + + const_reference front() const { + return TPages::front()->front(); + } + + reference back() { + return CurrentPage().back(); + } + + const_reference back() const { + return CurrentPage().back(); + } + + void clear() { + TPages::clear(); + } + + void resize(size_t sz) { + if (sz == size()) + return; + + const size_t npages = NPages(); + const size_t newwholepages = sz / PageSize; + const size_t pagepart = sz % PageSize; + const size_t newpages = newwholepages + bool(pagepart); + + if (npages && newwholepages >= npages) + CurrentPage().resize(PageSize); + + if (newpages < npages) + TPages::resize(newpages); + else + for (size_t i = npages; i < newpages; ++i) + MakeNewPage(); + + if (pagepart) + CurrentPage().resize(pagepart); + + Y_VERIFY(sz == size(), "%" PRIu64 " %" PRIu64, (ui64)sz, (ui64)size()); + } - for (size_t p = NPages() - 1; p > pnum; --p) { - PageAt(p).insert(PageAt(p).begin(), PageAt(p - 1).back()); - PageAt(p - 1).pop_back(); - } + reference at(size_t idx) { + return TPages::at(PageNumber(idx))->at(InPageIndex(idx)); + } - PageAt(pnum).insert(PageAt(pnum).begin() + pidx, v); - return it; - } + const_reference at(size_t idx) const { + return TPages::at(PageNumber(idx))->at(InPageIndex(idx)); + } - template <typename TIter> - void insert(iterator it, TIter b, TIter e) { - // todo : suboptimal! - for (; b != e; ++b, ++it) - it = insert(it, *b); - } + reference operator[](size_t idx) { + return TPages::operator[](PageNumber(idx))->operator[](InPageIndex(idx)); + } - reference front() { - return TPages::front()->front(); - } + const_reference operator[](size_t idx) const { + return TPages::operator[](PageNumber(idx))->operator[](InPageIndex(idx)); + } - const_reference front() const { - return TPages::front()->front(); - } + friend bool operator==(const TSelf& a, const TSelf& b) { + return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin()); + } - reference back() { - return CurrentPage().back(); - } - - const_reference back() const { - return CurrentPage().back(); - } - - void clear() { - TPages::clear(); - } - - void resize(size_t sz) { - if (sz == size()) - return; - - const size_t npages = NPages(); - const size_t newwholepages = sz / PageSize; - const size_t pagepart = sz % PageSize; - const size_t newpages = newwholepages + bool(pagepart); - - if (npages && newwholepages >= npages) - CurrentPage().resize(PageSize); - - if (newpages < npages) - TPages::resize(newpages); - else - for (size_t i = npages; i < newpages; ++i) - MakeNewPage(); - - if (pagepart) - CurrentPage().resize(pagepart); - - Y_VERIFY(sz == size(), "%" PRIu64 " %" PRIu64, (ui64)sz, (ui64)size()); - } - - reference at(size_t idx) { - return TPages::at(PageNumber(idx))->at(InPageIndex(idx)); - } - - const_reference at(size_t idx) const { - return TPages::at(PageNumber(idx))->at(InPageIndex(idx)); - } - - reference operator[](size_t idx) { - return TPages::operator[](PageNumber(idx))->operator[](InPageIndex(idx)); - } - - const_reference operator[](size_t idx) const { - return TPages::operator[](PageNumber(idx))->operator[](InPageIndex(idx)); - } - - friend bool operator==(const TSelf& a, const TSelf& b) { - return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin()); - } - - friend bool operator<(const TSelf& a, const TSelf& b) { - return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end()); - } - }; + friend bool operator<(const TSelf& a, const TSelf& b) { + return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end()); + } + }; - namespace NPrivate { - typedef std::is_same<std::random_access_iterator_tag, std::iterator_traits< - TPagedVector<ui32>::iterator>::iterator_category> - TIteratorCheck; - static_assert(TIteratorCheck::value, "expect TIteratorCheck::Result"); + namespace NPrivate { + typedef std::is_same<std::random_access_iterator_tag, std::iterator_traits< + TPagedVector<ui32>::iterator>::iterator_category> + TIteratorCheck; + static_assert(TIteratorCheck::value, "expect TIteratorCheck::Result"); } } diff --git a/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp b/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp index e867808ee4..2904a1d05e 100644 --- a/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp +++ b/library/cpp/containers/paged_vector/ut/paged_vector_ut.cpp @@ -1,378 +1,378 @@ #include <library/cpp/containers/paged_vector/paged_vector.h> #include <library/cpp/testing/unittest/registar.h> - -#include <stdexcept> + +#include <stdexcept> class TPagedVectorTest: public TTestBase { - UNIT_TEST_SUITE(TPagedVectorTest); - UNIT_TEST(Test0) - UNIT_TEST(Test1) - UNIT_TEST(Test2) - UNIT_TEST(Test3) - UNIT_TEST(Test4) - UNIT_TEST(Test5) - UNIT_TEST(Test6) - UNIT_TEST(Test7) - UNIT_TEST(TestAt) - UNIT_TEST(TestAutoRef) - UNIT_TEST(TestIterators) - //UNIT_TEST(TestEbo) - UNIT_TEST_SUITE_END(); - -private: - // Copy-paste of STLPort tests - void Test0() { - using NPagedVector::TPagedVector; - TPagedVector<int, 16> v1; // Empty vector of integers. - - UNIT_ASSERT(v1.empty() == true); - UNIT_ASSERT(v1.size() == 0); - - for (size_t i = 0; i < 256; ++i) { - v1.resize(i + 1); - UNIT_ASSERT_VALUES_EQUAL(v1.size(), i + 1); - } - - for (size_t i = 256; i-- > 0;) { - v1.resize(i); - UNIT_ASSERT_VALUES_EQUAL(v1.size(), i); + UNIT_TEST_SUITE(TPagedVectorTest); + UNIT_TEST(Test0) + UNIT_TEST(Test1) + UNIT_TEST(Test2) + UNIT_TEST(Test3) + UNIT_TEST(Test4) + UNIT_TEST(Test5) + UNIT_TEST(Test6) + UNIT_TEST(Test7) + UNIT_TEST(TestAt) + UNIT_TEST(TestAutoRef) + UNIT_TEST(TestIterators) + //UNIT_TEST(TestEbo) + UNIT_TEST_SUITE_END(); + +private: + // Copy-paste of STLPort tests + void Test0() { + using NPagedVector::TPagedVector; + TPagedVector<int, 16> v1; // Empty vector of integers. + + UNIT_ASSERT(v1.empty() == true); + UNIT_ASSERT(v1.size() == 0); + + for (size_t i = 0; i < 256; ++i) { + v1.resize(i + 1); + UNIT_ASSERT_VALUES_EQUAL(v1.size(), i + 1); + } + + for (size_t i = 256; i-- > 0;) { + v1.resize(i); + UNIT_ASSERT_VALUES_EQUAL(v1.size(), i); } - } + } - void Test1() { - using NPagedVector::TPagedVector; - TPagedVector<int, 3> v1; // Empty vector of integers. + void Test1() { + using NPagedVector::TPagedVector; + TPagedVector<int, 3> v1; // Empty vector of integers. - UNIT_ASSERT(v1.empty() == true); - UNIT_ASSERT(v1.size() == 0); + UNIT_ASSERT(v1.empty() == true); + UNIT_ASSERT(v1.size() == 0); - // UNIT_ASSERT(v1.max_size() == INT_MAX / sizeof(int)); - // cout << "max_size = " << v1.max_size() << endl; - v1.push_back(42); // Add an integer to the vector. + // UNIT_ASSERT(v1.max_size() == INT_MAX / sizeof(int)); + // cout << "max_size = " << v1.max_size() << endl; + v1.push_back(42); // Add an integer to the vector. - UNIT_ASSERT(v1.size() == 1); + UNIT_ASSERT(v1.size() == 1); - UNIT_ASSERT(v1[0] == 42); + UNIT_ASSERT(v1[0] == 42); - { - TPagedVector<TPagedVector<int, 3>, 3> vect; - vect.resize(10); - UNIT_ASSERT(vect.size() == 10); - TPagedVector<TPagedVector<int, 3>, 3>::iterator it(vect.begin()), end(vect.end()); - for (; it != end; ++it) { - UNIT_ASSERT((*it).empty()); - UNIT_ASSERT((*it).size() == 0); - UNIT_ASSERT((*it).begin() == (*it).end()); + { + TPagedVector<TPagedVector<int, 3>, 3> vect; + vect.resize(10); + UNIT_ASSERT(vect.size() == 10); + TPagedVector<TPagedVector<int, 3>, 3>::iterator it(vect.begin()), end(vect.end()); + for (; it != end; ++it) { + UNIT_ASSERT((*it).empty()); + UNIT_ASSERT((*it).size() == 0); + UNIT_ASSERT((*it).begin() == (*it).end()); } } - } + } - void Test2() { - using NPagedVector::TPagedVector; - TPagedVector<double, 3> v1; // Empty vector of doubles. - v1.push_back(32.1); - v1.push_back(40.5); - v1.push_back(45.5); - v1.push_back(33.4); - TPagedVector<double, 3> v2; // Another empty vector of doubles. - v2.push_back(3.56); + void Test2() { + using NPagedVector::TPagedVector; + TPagedVector<double, 3> v1; // Empty vector of doubles. + v1.push_back(32.1); + v1.push_back(40.5); + v1.push_back(45.5); + v1.push_back(33.4); + TPagedVector<double, 3> v2; // Another empty vector of doubles. + v2.push_back(3.56); - UNIT_ASSERT(v1.size() == 4); - UNIT_ASSERT(v1[0] == 32.1); - UNIT_ASSERT(v1[1] == 40.5); - UNIT_ASSERT(v1[2] == 45.5); - UNIT_ASSERT(v1[3] == 33.4); + UNIT_ASSERT(v1.size() == 4); + UNIT_ASSERT(v1[0] == 32.1); + UNIT_ASSERT(v1[1] == 40.5); + UNIT_ASSERT(v1[2] == 45.5); + UNIT_ASSERT(v1[3] == 33.4); - UNIT_ASSERT(v2.size() == 1); - UNIT_ASSERT(v2[0] == 3.56); - v1.swap(v2); // Swap the vector's contents. + UNIT_ASSERT(v2.size() == 1); + UNIT_ASSERT(v2[0] == 3.56); + v1.swap(v2); // Swap the vector's contents. - UNIT_ASSERT(v1.size() == 1); - UNIT_ASSERT(v1[0] == 3.56); + UNIT_ASSERT(v1.size() == 1); + UNIT_ASSERT(v1[0] == 3.56); - UNIT_ASSERT(v2.size() == 4); - UNIT_ASSERT(v2[0] == 32.1); - UNIT_ASSERT(v2[1] == 40.5); - UNIT_ASSERT(v2[2] == 45.5); - UNIT_ASSERT(v2[3] == 33.4); + UNIT_ASSERT(v2.size() == 4); + UNIT_ASSERT(v2[0] == 32.1); + UNIT_ASSERT(v2[1] == 40.5); + UNIT_ASSERT(v2[2] == 45.5); + UNIT_ASSERT(v2[3] == 33.4); - v2 = v1; // Assign one vector to another. + v2 = v1; // Assign one vector to another. - UNIT_ASSERT(v2.size() == 1); - UNIT_ASSERT(v2[0] == 3.56); + UNIT_ASSERT(v2.size() == 1); + UNIT_ASSERT(v2[0] == 3.56); - v2.pop_back(); - UNIT_ASSERT(v2.size() == 0); - UNIT_ASSERT(v2.empty()); - } + v2.pop_back(); + UNIT_ASSERT(v2.size() == 0); + UNIT_ASSERT(v2.empty()); + } - void Test3() { - using NPagedVector::TPagedVector; - TPagedVector<char, 1> v1; + void Test3() { + using NPagedVector::TPagedVector; + TPagedVector<char, 1> v1; - v1.push_back('h'); - v1.push_back('i'); + v1.push_back('h'); + v1.push_back('i'); - UNIT_ASSERT(v1.size() == 2); - UNIT_ASSERT(v1[0] == 'h'); - UNIT_ASSERT(v1[1] == 'i'); + UNIT_ASSERT(v1.size() == 2); + UNIT_ASSERT(v1[0] == 'h'); + UNIT_ASSERT(v1[1] == 'i'); - TPagedVector<char, 1> v2; - v2.resize(v1.size()); + TPagedVector<char, 1> v2; + v2.resize(v1.size()); - for (size_t i = 0; i < v1.size(); ++i) - v2[i] = v1[i]; + for (size_t i = 0; i < v1.size(); ++i) + v2[i] = v1[i]; - v2[1] = 'o'; // Replace second character. + v2[1] = 'o'; // Replace second character. - UNIT_ASSERT(v2.size() == 2); - UNIT_ASSERT(v2[0] == 'h'); - UNIT_ASSERT(v2[1] == 'o'); + UNIT_ASSERT(v2.size() == 2); + UNIT_ASSERT(v2[0] == 'h'); + UNIT_ASSERT(v2[1] == 'o'); - UNIT_ASSERT((v1 == v2) == false); + UNIT_ASSERT((v1 == v2) == false); - UNIT_ASSERT((v1 < v2) == true); - } + UNIT_ASSERT((v1 < v2) == true); + } - void Test4() { - using NPagedVector::TPagedVector; - TPagedVector<int, 3> v; - v.resize(4); + void Test4() { + using NPagedVector::TPagedVector; + TPagedVector<int, 3> v; + v.resize(4); - v[0] = 1; - v[1] = 4; - v[2] = 9; - v[3] = 16; + v[0] = 1; + v[1] = 4; + v[2] = 9; + v[3] = 16; - UNIT_ASSERT(v.front() == 1); - UNIT_ASSERT(v.back() == 16); + UNIT_ASSERT(v.front() == 1); + UNIT_ASSERT(v.back() == 16); - v.push_back(25); + v.push_back(25); - UNIT_ASSERT(v.back() == 25); - UNIT_ASSERT(v.size() == 5); + UNIT_ASSERT(v.back() == 25); + UNIT_ASSERT(v.size() == 5); - v.pop_back(); + v.pop_back(); - UNIT_ASSERT(v.back() == 16); - UNIT_ASSERT(v.size() == 4); - } + UNIT_ASSERT(v.back() == 16); + UNIT_ASSERT(v.size() == 4); + } - void Test5() { - int array[] = {1, 4, 9, 16}; + void Test5() { + int array[] = {1, 4, 9, 16}; - typedef NPagedVector::TPagedVector<int, 3> TVectorType; - TVectorType v(array, array + 4); + typedef NPagedVector::TPagedVector<int, 3> TVectorType; + TVectorType v(array, array + 4); - UNIT_ASSERT(v.size() == 4); - - UNIT_ASSERT(v[0] == 1); - UNIT_ASSERT(v[1] == 4); - UNIT_ASSERT(v[2] == 9); - UNIT_ASSERT(v[3] == 16); - } - - void Test6() { - int array[] = {1, 4, 9, 16, 25, 36}; + UNIT_ASSERT(v.size() == 4); + + UNIT_ASSERT(v[0] == 1); + UNIT_ASSERT(v[1] == 4); + UNIT_ASSERT(v[2] == 9); + UNIT_ASSERT(v[3] == 16); + } + + void Test6() { + int array[] = {1, 4, 9, 16, 25, 36}; - typedef NPagedVector::TPagedVector<int, 3> TVectorType; - TVectorType v(array, array + 6); - TVectorType::iterator vit; + typedef NPagedVector::TPagedVector<int, 3> TVectorType; + TVectorType v(array, array + 6); + TVectorType::iterator vit; - UNIT_ASSERT_VALUES_EQUAL(v.size(), 6u); - UNIT_ASSERT(v[0] == 1); - UNIT_ASSERT(v[1] == 4); - UNIT_ASSERT(v[2] == 9); - UNIT_ASSERT(v[3] == 16); - UNIT_ASSERT(v[4] == 25); - UNIT_ASSERT(v[5] == 36); + UNIT_ASSERT_VALUES_EQUAL(v.size(), 6u); + UNIT_ASSERT(v[0] == 1); + UNIT_ASSERT(v[1] == 4); + UNIT_ASSERT(v[2] == 9); + UNIT_ASSERT(v[3] == 16); + UNIT_ASSERT(v[4] == 25); + UNIT_ASSERT(v[5] == 36); - vit = v.erase(v.begin()); // Erase first element. - UNIT_ASSERT(*vit == 4); + vit = v.erase(v.begin()); // Erase first element. + UNIT_ASSERT(*vit == 4); - UNIT_ASSERT(v.size() == 5); - UNIT_ASSERT(v[0] == 4); - UNIT_ASSERT(v[1] == 9); - UNIT_ASSERT(v[2] == 16); - UNIT_ASSERT(v[3] == 25); - UNIT_ASSERT(v[4] == 36); + UNIT_ASSERT(v.size() == 5); + UNIT_ASSERT(v[0] == 4); + UNIT_ASSERT(v[1] == 9); + UNIT_ASSERT(v[2] == 16); + UNIT_ASSERT(v[3] == 25); + UNIT_ASSERT(v[4] == 36); - vit = v.erase(v.end() - 1); // Erase last element. - UNIT_ASSERT(vit == v.end()); + vit = v.erase(v.end() - 1); // Erase last element. + UNIT_ASSERT(vit == v.end()); - UNIT_ASSERT(v.size() == 4); - UNIT_ASSERT(v[0] == 4); - UNIT_ASSERT(v[1] == 9); - UNIT_ASSERT(v[2] == 16); - UNIT_ASSERT(v[3] == 25); + UNIT_ASSERT(v.size() == 4); + UNIT_ASSERT(v[0] == 4); + UNIT_ASSERT(v[1] == 9); + UNIT_ASSERT(v[2] == 16); + UNIT_ASSERT(v[3] == 25); - v.erase(v.begin() + 1, v.end() - 1); // Erase all but first and last. + v.erase(v.begin() + 1, v.end() - 1); // Erase all but first and last. - UNIT_ASSERT(v.size() == 2); - UNIT_ASSERT(v[0] == 4); - UNIT_ASSERT(v[1] == 25); - } + UNIT_ASSERT(v.size() == 2); + UNIT_ASSERT(v[0] == 4); + UNIT_ASSERT(v[1] == 25); + } - void Test7() { - int array1[] = {1, 4, 25}; - int array2[] = {9, 16}; + void Test7() { + int array1[] = {1, 4, 25}; + int array2[] = {9, 16}; - typedef NPagedVector::TPagedVector<int, 3> TVectorType; + typedef NPagedVector::TPagedVector<int, 3> TVectorType; - TVectorType v(array1, array1 + 3); - TVectorType::iterator vit; - vit = v.insert(v.begin(), 0); // Insert before first element. - UNIT_ASSERT_VALUES_EQUAL(*vit, 0); - - vit = v.insert(v.end(), 36); // Insert after last element. - UNIT_ASSERT(*vit == 36); - - UNIT_ASSERT(v.size() == 5); - UNIT_ASSERT(v[0] == 0); - UNIT_ASSERT(v[1] == 1); - UNIT_ASSERT(v[2] == 4); - UNIT_ASSERT(v[3] == 25); - UNIT_ASSERT(v[4] == 36); - - // Insert contents of array2 before fourth element. - v.insert(v.begin() + 3, array2, array2 + 2); - - UNIT_ASSERT(v.size() == 7); - - UNIT_ASSERT(v[0] == 0); - UNIT_ASSERT(v[1] == 1); - UNIT_ASSERT(v[2] == 4); - UNIT_ASSERT(v[3] == 9); - UNIT_ASSERT(v[4] == 16); - UNIT_ASSERT(v[5] == 25); - UNIT_ASSERT(v[6] == 36); - - v.clear(); - UNIT_ASSERT(v.empty()); - } - - void TestAt() { - using NPagedVector::TPagedVector; - TPagedVector<int, 3> v; - TPagedVector<int, 3> const& cv = v; - - v.push_back(10); - UNIT_ASSERT(v.at(0) == 10); - v.at(0) = 20; - UNIT_ASSERT(cv.at(0) == 20); - - for (;;) { - try { - v.at(1) = 20; - UNIT_ASSERT(false); - } catch (std::out_of_range const&) { - return; - } catch (...) { - UNIT_ASSERT(false); + TVectorType v(array1, array1 + 3); + TVectorType::iterator vit; + vit = v.insert(v.begin(), 0); // Insert before first element. + UNIT_ASSERT_VALUES_EQUAL(*vit, 0); + + vit = v.insert(v.end(), 36); // Insert after last element. + UNIT_ASSERT(*vit == 36); + + UNIT_ASSERT(v.size() == 5); + UNIT_ASSERT(v[0] == 0); + UNIT_ASSERT(v[1] == 1); + UNIT_ASSERT(v[2] == 4); + UNIT_ASSERT(v[3] == 25); + UNIT_ASSERT(v[4] == 36); + + // Insert contents of array2 before fourth element. + v.insert(v.begin() + 3, array2, array2 + 2); + + UNIT_ASSERT(v.size() == 7); + + UNIT_ASSERT(v[0] == 0); + UNIT_ASSERT(v[1] == 1); + UNIT_ASSERT(v[2] == 4); + UNIT_ASSERT(v[3] == 9); + UNIT_ASSERT(v[4] == 16); + UNIT_ASSERT(v[5] == 25); + UNIT_ASSERT(v[6] == 36); + + v.clear(); + UNIT_ASSERT(v.empty()); + } + + void TestAt() { + using NPagedVector::TPagedVector; + TPagedVector<int, 3> v; + TPagedVector<int, 3> const& cv = v; + + v.push_back(10); + UNIT_ASSERT(v.at(0) == 10); + v.at(0) = 20; + UNIT_ASSERT(cv.at(0) == 20); + + for (;;) { + try { + v.at(1) = 20; + UNIT_ASSERT(false); + } catch (std::out_of_range const&) { + return; + } catch (...) { + UNIT_ASSERT(false); } } - } - - void TestAutoRef() { - using NPagedVector::TPagedVector; - typedef TPagedVector<int, 3> TVec; - TVec ref; - for (int i = 0; i < 5; ++i) { - ref.push_back(i); - } - - TPagedVector<TVec, 3> v_v_int; - v_v_int.push_back(ref); - v_v_int.push_back(v_v_int[0]); - v_v_int.push_back(ref); - v_v_int.push_back(v_v_int[0]); - v_v_int.push_back(v_v_int[0]); - v_v_int.push_back(ref); - - TPagedVector<TVec, 3>::iterator vvit(v_v_int.begin()), vvitEnd(v_v_int.end()); - for (; vvit != vvitEnd; ++vvit) { - UNIT_ASSERT(*vvit == ref); - } - } - - struct Point { - int x, y; - }; - - struct PointEx: public Point { - PointEx() - : builtFromBase(false) - { + } + + void TestAutoRef() { + using NPagedVector::TPagedVector; + typedef TPagedVector<int, 3> TVec; + TVec ref; + for (int i = 0; i < 5; ++i) { + ref.push_back(i); + } + + TPagedVector<TVec, 3> v_v_int; + v_v_int.push_back(ref); + v_v_int.push_back(v_v_int[0]); + v_v_int.push_back(ref); + v_v_int.push_back(v_v_int[0]); + v_v_int.push_back(v_v_int[0]); + v_v_int.push_back(ref); + + TPagedVector<TVec, 3>::iterator vvit(v_v_int.begin()), vvitEnd(v_v_int.end()); + for (; vvit != vvitEnd; ++vvit) { + UNIT_ASSERT(*vvit == ref); } - PointEx(const Point&) - : builtFromBase(true) - { - } - - bool builtFromBase; - }; - - void TestIterators() { - using NPagedVector::TPagedVector; - TPagedVector<int, 3> vint; - vint.resize(10); - TPagedVector<int, 3> const& crvint = vint; - - UNIT_ASSERT(vint.begin() == vint.begin()); - UNIT_ASSERT(crvint.begin() == vint.begin()); - UNIT_ASSERT(vint.begin() == crvint.begin()); - UNIT_ASSERT(crvint.begin() == crvint.begin()); - - UNIT_ASSERT(vint.begin() != vint.end()); - UNIT_ASSERT(crvint.begin() != vint.end()); - UNIT_ASSERT(vint.begin() != crvint.end()); - UNIT_ASSERT(crvint.begin() != crvint.end()); - - UNIT_ASSERT(vint.rbegin() == vint.rbegin()); - // Not Standard: - //UNIT_ASSERT(vint.rbegin() == crvint.rbegin()); - //UNIT_ASSERT(crvint.rbegin() == vint.rbegin()); - UNIT_ASSERT(crvint.rbegin() == crvint.rbegin()); - - UNIT_ASSERT(vint.rbegin() != vint.rend()); - // Not Standard: - //UNIT_ASSERT(vint.rbegin() != crvint.rend()); - //UNIT_ASSERT(crvint.rbegin() != vint.rend()); - UNIT_ASSERT(crvint.rbegin() != crvint.rend()); - } - - /* This test check a potential issue with empty base class + } + + struct Point { + int x, y; + }; + + struct PointEx: public Point { + PointEx() + : builtFromBase(false) + { + } + PointEx(const Point&) + : builtFromBase(true) + { + } + + bool builtFromBase; + }; + + void TestIterators() { + using NPagedVector::TPagedVector; + TPagedVector<int, 3> vint; + vint.resize(10); + TPagedVector<int, 3> const& crvint = vint; + + UNIT_ASSERT(vint.begin() == vint.begin()); + UNIT_ASSERT(crvint.begin() == vint.begin()); + UNIT_ASSERT(vint.begin() == crvint.begin()); + UNIT_ASSERT(crvint.begin() == crvint.begin()); + + UNIT_ASSERT(vint.begin() != vint.end()); + UNIT_ASSERT(crvint.begin() != vint.end()); + UNIT_ASSERT(vint.begin() != crvint.end()); + UNIT_ASSERT(crvint.begin() != crvint.end()); + + UNIT_ASSERT(vint.rbegin() == vint.rbegin()); + // Not Standard: + //UNIT_ASSERT(vint.rbegin() == crvint.rbegin()); + //UNIT_ASSERT(crvint.rbegin() == vint.rbegin()); + UNIT_ASSERT(crvint.rbegin() == crvint.rbegin()); + + UNIT_ASSERT(vint.rbegin() != vint.rend()); + // Not Standard: + //UNIT_ASSERT(vint.rbegin() != crvint.rend()); + //UNIT_ASSERT(crvint.rbegin() != vint.rend()); + UNIT_ASSERT(crvint.rbegin() != crvint.rend()); + } + + /* This test check a potential issue with empty base class * optimization. Some compilers (VC6) do not implement it * correctly resulting ina wrong behavior. */ - void TestEbo() { - using NPagedVector::TPagedVector; - // We use heap memory as test failure can corrupt vector internal - // representation making executable crash on vector destructor invocation. - // We prefer a simple memory leak, internal corruption should be reveal - // by size or capacity checks. - typedef TPagedVector<int, 3> V; - V* pv1 = new V; + void TestEbo() { + using NPagedVector::TPagedVector; + // We use heap memory as test failure can corrupt vector internal + // representation making executable crash on vector destructor invocation. + // We prefer a simple memory leak, internal corruption should be reveal + // by size or capacity checks. + typedef TPagedVector<int, 3> V; + V* pv1 = new V; - pv1->resize(1); - pv1->at(0) = 1; + pv1->resize(1); + pv1->at(0) = 1; - V* pv2 = new V; + V* pv2 = new V; - pv2->resize(10); - for (int i = 0; i < 10; ++i) - pv2->at(i) = 2; + pv2->resize(10); + for (int i = 0; i < 10; ++i) + pv2->at(i) = 2; - pv1->swap(*pv2); + pv1->swap(*pv2); - UNIT_ASSERT(pv1->size() == 10); - UNIT_ASSERT((*pv1)[5] == 2); + UNIT_ASSERT(pv1->size() == 10); + UNIT_ASSERT((*pv1)[5] == 2); - UNIT_ASSERT(pv2->size() == 1); - UNIT_ASSERT((*pv2)[0] == 1); + UNIT_ASSERT(pv2->size() == 1); + UNIT_ASSERT((*pv2)[0] == 1); - delete pv2; - delete pv1; - } + delete pv2; + delete pv1; + } }; UNIT_TEST_SUITE_REGISTRATION(TPagedVectorTest); diff --git a/library/cpp/containers/paged_vector/ut/ya.make b/library/cpp/containers/paged_vector/ut/ya.make index 74cfe5fb4a..69054531b1 100644 --- a/library/cpp/containers/paged_vector/ut/ya.make +++ b/library/cpp/containers/paged_vector/ut/ya.make @@ -1,13 +1,13 @@ -UNITTEST() - -OWNER(velavokr) - -PEERDIR( +UNITTEST() + +OWNER(velavokr) + +PEERDIR( library/cpp/containers/paged_vector -) - -SRCS( - paged_vector_ut.cpp -) - -END() +) + +SRCS( + paged_vector_ut.cpp +) + +END() diff --git a/library/cpp/containers/sorted_vector/sorted_vector.cpp b/library/cpp/containers/sorted_vector/sorted_vector.cpp index 56aaf69dde..806efe371a 100644 --- a/library/cpp/containers/sorted_vector/sorted_vector.cpp +++ b/library/cpp/containers/sorted_vector/sorted_vector.cpp @@ -1 +1 @@ -#include "sorted_vector.h" +#include "sorted_vector.h" diff --git a/library/cpp/containers/sorted_vector/sorted_vector.h b/library/cpp/containers/sorted_vector/sorted_vector.h index 123539af9e..17a1fddcc5 100644 --- a/library/cpp/containers/sorted_vector/sorted_vector.h +++ b/library/cpp/containers/sorted_vector/sorted_vector.h @@ -1,85 +1,85 @@ -#pragma once - -#include <util/system/defaults.h> -#include <util/generic/hash.h> -#include <util/generic/vector.h> -#include <util/generic/algorithm.h> +#pragma once + +#include <util/system/defaults.h> +#include <util/generic/hash.h> +#include <util/generic/vector.h> +#include <util/generic/algorithm.h> #include <util/generic/mapfindptr.h> -#include <util/ysaveload.h> +#include <util/ysaveload.h> #include <utility> - + #include <initializer_list> -namespace NSorted { - namespace NPrivate { +namespace NSorted { + namespace NPrivate { template <class TPredicate> struct TEqual { template<typename TValueType1, typename TValueType2> inline bool operator()(const TValueType1& l, const TValueType2& r) const { - TPredicate comp; - return comp(l, r) == comp(r, l); - } - }; - + TPredicate comp; + return comp(l, r) == comp(r, l); + } + }; + template <typename TValueType, class TPredicate, class TKeyExtractor> struct TKeyCompare { - inline bool operator()(const TValueType& l, const TValueType& r) const { - TKeyExtractor extractKey; - return TPredicate()(extractKey(l), extractKey(r)); - } + inline bool operator()(const TValueType& l, const TValueType& r) const { + TKeyExtractor extractKey; + return TPredicate()(extractKey(l), extractKey(r)); + } template<typename TKeyType> - inline bool operator()(const TKeyType& l, const TValueType& r) const { - return TPredicate()(l, TKeyExtractor()(r)); - } + inline bool operator()(const TKeyType& l, const TValueType& r) const { + return TPredicate()(l, TKeyExtractor()(r)); + } template<typename TKeyType> - inline bool operator()(const TValueType& l, const TKeyType& r) const { - return TPredicate()(TKeyExtractor()(l), r); - } - }; - - template <typename TValueType, class TPredicate> + inline bool operator()(const TValueType& l, const TKeyType& r) const { + return TPredicate()(TKeyExtractor()(l), r); + } + }; + + template <typename TValueType, class TPredicate> struct TKeyCompare<TValueType, TPredicate, TIdentity> { template <typename TValueType1, typename TValueType2> inline bool operator()(const TValueType1& l, const TValueType2& r) const { - return TPredicate()(l, r); - } - }; - - } - - // Sorted vector, which is order by the key. The key is extracted from the value by the provided key-extractor - template <typename TValueType, typename TKeyType = TValueType, class TKeyExtractor = TIdentity, + return TPredicate()(l, r); + } + }; + + } + + // Sorted vector, which is order by the key. The key is extracted from the value by the provided key-extractor + template <typename TValueType, typename TKeyType = TValueType, class TKeyExtractor = TIdentity, class TPredicate = TLess<TKeyType>, class A = std::allocator<TValueType>> class TSortedVector: public TVector<TValueType, A> { - private: + private: typedef TVector<TValueType, A> TBase; typedef NPrivate::TKeyCompare<TValueType, TPredicate, TKeyExtractor> TKeyCompare; typedef NPrivate::TEqual<TKeyCompare> TValueEqual; typedef NPrivate::TEqual<TPredicate> TKeyEqual; - - public: - typedef TValueType value_type; - typedef TKeyType key_type; - typedef typename TBase::iterator iterator; - typedef typename TBase::const_iterator const_iterator; - typedef typename TBase::size_type size_type; - - public: - inline TSortedVector() - : TBase() - { - } - - inline explicit TSortedVector(size_type count) - : TBase(count) - { - } - - inline TSortedVector(size_type count, const value_type& val) - : TBase(count, val) - { - } - + + public: + typedef TValueType value_type; + typedef TKeyType key_type; + typedef typename TBase::iterator iterator; + typedef typename TBase::const_iterator const_iterator; + typedef typename TBase::size_type size_type; + + public: + inline TSortedVector() + : TBase() + { + } + + inline explicit TSortedVector(size_type count) + : TBase(count) + { + } + + inline TSortedVector(size_type count, const value_type& val) + : TBase(count, val) + { + } + inline TSortedVector(std::initializer_list<value_type> il) : TBase(il) { @@ -92,293 +92,293 @@ namespace NSorted { Sort(); } - template <class TIter> - inline TSortedVector(TIter first, TIter last) - : TBase(first, last) - { - Sort(); - } - - // Inserts non-unique value in the proper position according to the key-sort order. - // Returns iterator, which points to the inserted value - inline iterator Insert(const value_type& value) { - return TBase::insert(LowerBound(TKeyExtractor()(value)), value); - } - - // STL-compatible synonym + template <class TIter> + inline TSortedVector(TIter first, TIter last) + : TBase(first, last) + { + Sort(); + } + + // Inserts non-unique value in the proper position according to the key-sort order. + // Returns iterator, which points to the inserted value + inline iterator Insert(const value_type& value) { + return TBase::insert(LowerBound(TKeyExtractor()(value)), value); + } + + // STL-compatible synonym Y_FORCE_INLINE iterator insert(const value_type& value) { - return this->Insert(value); - } - - // Inserts non-unique value range in the proper position according to the key-sort order. - template <class TIter> - inline void Insert(TIter first, TIter last) { - TBase::insert(TBase::end(), first, last); - Sort(); - } - - // STL-compatible synonym - template <class TIter> + return this->Insert(value); + } + + // Inserts non-unique value range in the proper position according to the key-sort order. + template <class TIter> + inline void Insert(TIter first, TIter last) { + TBase::insert(TBase::end(), first, last); + Sort(); + } + + // STL-compatible synonym + template <class TIter> Y_FORCE_INLINE void insert(TIter first, TIter last) { - this->Insert(first, last); - } - - // Inserts unique value in the proper position according to the key-sort order, - // if the value with the same key doesn't exist. Returns <iterator, bool> pair, - // where the first member is the pointer to the inserted/existing value, and the - // second member indicates either the value is inserted or not. + this->Insert(first, last); + } + + // Inserts unique value in the proper position according to the key-sort order, + // if the value with the same key doesn't exist. Returns <iterator, bool> pair, + // where the first member is the pointer to the inserted/existing value, and the + // second member indicates either the value is inserted or not. inline std::pair<iterator, bool> InsertUnique(const value_type& value) { - iterator i = LowerBound(TKeyExtractor()(value)); - if (i == TBase::end() || !TValueEqual()(*i, value)) + iterator i = LowerBound(TKeyExtractor()(value)); + if (i == TBase::end() || !TValueEqual()(*i, value)) return std::pair<iterator, bool>(TBase::insert(i, value), true); - else + else return std::pair<iterator, bool>(i, false); - } - - // STL-compatible synonym + } + + // STL-compatible synonym Y_FORCE_INLINE std::pair<iterator, bool> insert_unique(const value_type& value) { - return this->InsertUnique(value); - } - - // Inserts unique value range in the proper position according to the key-sort order. - template <class TIter> - inline void InsertUnique(TIter first, TIter last) { - TBase::insert(TBase::end(), first, last); - Sort(); - MakeUnique(); - } - - // STL-compatible synonym - template <class TIter> + return this->InsertUnique(value); + } + + // Inserts unique value range in the proper position according to the key-sort order. + template <class TIter> + inline void InsertUnique(TIter first, TIter last) { + TBase::insert(TBase::end(), first, last); + Sort(); + MakeUnique(); + } + + // STL-compatible synonym + template <class TIter> Y_FORCE_INLINE void insert_unique(TIter first, TIter last) { - this->InsertUnique(first, last); - } - - // Inserts unique value in the proper position according to the key-sort order. - // If the value with the same key already exists, then it is replaced with the new one. - // Returns iterator, which points to the inserted value - inline iterator InsertOrReplace(const value_type& value) { - iterator i = ::LowerBound(TBase::begin(), TBase::end(), value, TKeyCompare()); - if (i == TBase::end() || !TValueEqual()(*i, value)) - return TBase::insert(i, value); - else - return TBase::insert(TBase::erase(i), value); - } - - // STL-compatible synonym + this->InsertUnique(first, last); + } + + // Inserts unique value in the proper position according to the key-sort order. + // If the value with the same key already exists, then it is replaced with the new one. + // Returns iterator, which points to the inserted value + inline iterator InsertOrReplace(const value_type& value) { + iterator i = ::LowerBound(TBase::begin(), TBase::end(), value, TKeyCompare()); + if (i == TBase::end() || !TValueEqual()(*i, value)) + return TBase::insert(i, value); + else + return TBase::insert(TBase::erase(i), value); + } + + // STL-compatible synonym Y_FORCE_INLINE iterator insert_or_replace(const value_type& value) { - return this->InsertOrReplace(value); - } - + return this->InsertOrReplace(value); + } + Y_FORCE_INLINE void Sort() { - ::Sort(TBase::begin(), TBase::end(), TKeyCompare()); - } - - // STL-compatible synonym + ::Sort(TBase::begin(), TBase::end(), TKeyCompare()); + } + + // STL-compatible synonym Y_FORCE_INLINE void sort() { - this->Sort(); - } - + this->Sort(); + } + Y_FORCE_INLINE void Sort(iterator from, iterator to) { - ::Sort(from, to, TKeyCompare()); - } - - // STL-compatible synonym + ::Sort(from, to, TKeyCompare()); + } + + // STL-compatible synonym Y_FORCE_INLINE void sort(iterator from, iterator to) { - this->Sort(from, to); - } - - inline void MakeUnique() { - TBase::erase(::Unique(TBase::begin(), TBase::end(), TValueEqual()), TBase::end()); - } - - // STL-compatible synonym + this->Sort(from, to); + } + + inline void MakeUnique() { + TBase::erase(::Unique(TBase::begin(), TBase::end(), TValueEqual()), TBase::end()); + } + + // STL-compatible synonym Y_FORCE_INLINE void make_unique() { - this->MakeUnique(); - } - + this->MakeUnique(); + } + template<class K> inline const_iterator Find(const K& key) const { - const_iterator i = LowerBound(key); - if (i == TBase::end() || !TKeyEqual()(TKeyExtractor()(*i), key)) - return TBase::end(); - else - return i; - } - - // STL-compatible synonym + const_iterator i = LowerBound(key); + if (i == TBase::end() || !TKeyEqual()(TKeyExtractor()(*i), key)) + return TBase::end(); + else + return i; + } + + // STL-compatible synonym template<class K> Y_FORCE_INLINE const_iterator find(const K& key) const { - return this->Find(key); - } - + return this->Find(key); + } + template<class K> inline iterator Find(const K& key) { - iterator i = LowerBound(key); - if (i == TBase::end() || !TKeyEqual()(TKeyExtractor()(*i), key)) - return TBase::end(); - else - return i; - } - - // STL-compatible synonym + iterator i = LowerBound(key); + if (i == TBase::end() || !TKeyEqual()(TKeyExtractor()(*i), key)) + return TBase::end(); + else + return i; + } + + // STL-compatible synonym template<class K> Y_FORCE_INLINE iterator find(const K& key) { - return this->Find(key); - } - + return this->Find(key); + } + template<class K> Y_FORCE_INLINE bool Has(const K& key) const { - return this->find(key) != TBase::end(); - } - + return this->find(key) != TBase::end(); + } + template<class K> Y_FORCE_INLINE bool has(const K& key) const { - return this->Has(key); - } - + return this->Has(key); + } + template<class K> Y_FORCE_INLINE iterator LowerBound(const K& key) { - return ::LowerBound(TBase::begin(), TBase::end(), key, TKeyCompare()); - } - - // STL-compatible synonym + return ::LowerBound(TBase::begin(), TBase::end(), key, TKeyCompare()); + } + + // STL-compatible synonym template<class K> Y_FORCE_INLINE iterator lower_bound(const K& key) { - return this->LowerBound(key); - } - + return this->LowerBound(key); + } + template<class K> Y_FORCE_INLINE const_iterator LowerBound(const K& key) const { - return ::LowerBound(TBase::begin(), TBase::end(), key, TKeyCompare()); - } - - // STL-compatible synonym + return ::LowerBound(TBase::begin(), TBase::end(), key, TKeyCompare()); + } + + // STL-compatible synonym template<class K> Y_FORCE_INLINE const_iterator lower_bound(const K& key) const { - return this->LowerBound(key); - } - + return this->LowerBound(key); + } + template<class K> Y_FORCE_INLINE iterator UpperBound(const K& key) { - return ::UpperBound(TBase::begin(), TBase::end(), key, TKeyCompare()); - } - - // STL-compatible synonym + return ::UpperBound(TBase::begin(), TBase::end(), key, TKeyCompare()); + } + + // STL-compatible synonym template<class K> Y_FORCE_INLINE iterator upper_bound(const K& key) { - return this->UpperBound(key); - } - + return this->UpperBound(key); + } + template<class K> Y_FORCE_INLINE const_iterator UpperBound(const K& key) const { - return ::UpperBound(TBase::begin(), TBase::end(), key, TKeyCompare()); - } - - // STL-compatible synonym + return ::UpperBound(TBase::begin(), TBase::end(), key, TKeyCompare()); + } + + // STL-compatible synonym template<class K> Y_FORCE_INLINE const_iterator upper_bound(const K& key) const { - return this->UpperBound(key); - } - + return this->UpperBound(key); + } + template<class K> Y_FORCE_INLINE std::pair<iterator, iterator> EqualRange(const K& key) { - return std::equal_range(TBase::begin(), TBase::end(), key, TKeyCompare()); - } - - // STL-compatible synonym + return std::equal_range(TBase::begin(), TBase::end(), key, TKeyCompare()); + } + + // STL-compatible synonym template<class K> Y_FORCE_INLINE std::pair<iterator, iterator> equal_range(const K& key) { - return this->EqualRange(key); - } - + return this->EqualRange(key); + } + template<class K> Y_FORCE_INLINE std::pair<const_iterator, const_iterator> EqualRange(const K& key) const { - return std::equal_range(TBase::begin(), TBase::end(), key, TKeyCompare()); - } - - // STL-compatible synonym + return std::equal_range(TBase::begin(), TBase::end(), key, TKeyCompare()); + } + + // STL-compatible synonym template<class K> Y_FORCE_INLINE std::pair<const_iterator, const_iterator> equal_range(const K& key) const { - return this->EqualRange(key); - } - + return this->EqualRange(key); + } + template<class K> inline void Erase(const K& key) { std::pair<iterator, iterator> res = EqualRange(key); - TBase::erase(res.first, res.second); - } - - // STL-compatible synonym + TBase::erase(res.first, res.second); + } + + // STL-compatible synonym Y_FORCE_INLINE void erase(const key_type& key) { - this->Erase(key); - } - + this->Erase(key); + } + template<class K> inline size_t count(const K& key) const { const std::pair<const_iterator, const_iterator> range = this->EqualRange(key); - return std::distance(range.first, range.second); - } - - using TBase::erase; - }; - + return std::distance(range.first, range.second); + } + + using TBase::erase; + }; + // The simplified map (a.k.a TFlatMap, flat_map), which is implemented by the sorted-vector. - // This structure has the side-effect: if you keep a reference to an existing element - // and then inserts a new one, the existing reference can be broken (due to reallocation). - // Please keep this in mind when using this structure. + // This structure has the side-effect: if you keep a reference to an existing element + // and then inserts a new one, the existing reference can be broken (due to reallocation). + // Please keep this in mind when using this structure. template <typename TKeyType, typename TValueType, class TPredicate = TLess<TKeyType>, class A = std::allocator<TValueType>> class TSimpleMap: public TSortedVector<std::pair<TKeyType, TValueType>, TKeyType, TSelect1st, TPredicate, A>, public TMapOps<TSimpleMap<TKeyType, TValueType, TPredicate, A>> { - private: + private: typedef TSortedVector<std::pair<TKeyType, TValueType>, TKeyType, TSelect1st, TPredicate, A> TBase; - - public: - typedef typename TBase::value_type value_type; - typedef typename TBase::key_type key_type; - typedef typename TBase::iterator iterator; - typedef typename TBase::const_iterator const_iterator; - typedef typename TBase::size_type size_type; - - public: - inline TSimpleMap() - : TBase() - { - } - - template <class TIter> - inline TSimpleMap(TIter first, TIter last) - : TBase(first, last) - { - TBase::MakeUnique(); - } - + + public: + typedef typename TBase::value_type value_type; + typedef typename TBase::key_type key_type; + typedef typename TBase::iterator iterator; + typedef typename TBase::const_iterator const_iterator; + typedef typename TBase::size_type size_type; + + public: + inline TSimpleMap() + : TBase() + { + } + + template <class TIter> + inline TSimpleMap(TIter first, TIter last) + : TBase(first, last) + { + TBase::MakeUnique(); + } + inline TSimpleMap(std::initializer_list<value_type> il) : TBase(il) { TBase::MakeUnique(); } - inline TValueType& Get(const TKeyType& key) { - typename TBase::iterator i = TBase::LowerBound(key); - if (i == TBase::end() || key != i->first) + inline TValueType& Get(const TKeyType& key) { + typename TBase::iterator i = TBase::LowerBound(key); + if (i == TBase::end() || key != i->first) return TVector<std::pair<TKeyType, TValueType>, A>::insert(i, std::make_pair(key, TValueType()))->second; - else - return i->second; - } - + else + return i->second; + } + template<class K> inline const TValueType& Get(const K& key, const TValueType& def) const { - typename TBase::const_iterator i = TBase::Find(key); - return i != TBase::end() ? i->second : def; - } - + typename TBase::const_iterator i = TBase::Find(key); + return i != TBase::end() ? i->second : def; + } + template<class K> Y_FORCE_INLINE TValueType& operator[](const K& key) { - return Get(key); - } + return Get(key); + } template<class K> const TValueType& at(const K& key) const { @@ -395,98 +395,98 @@ namespace NSorted { return const_cast<TValueType&>( const_cast<const TSimpleMap<TKeyType, TValueType, TPredicate, A>*>(this)->at(key)); } - }; - + }; + // The simplified set (a.k.a TFlatSet, flat_set), which is implemented by the sorted-vector. - // This structure has the same side-effect as TSimpleMap. - // The value type must have TValueType(TKeyType) constructor in order to use [] operator - template <typename TValueType, typename TKeyType = TValueType, class TKeyExtractor = TIdentity, + // This structure has the same side-effect as TSimpleMap. + // The value type must have TValueType(TKeyType) constructor in order to use [] operator + template <typename TValueType, typename TKeyType = TValueType, class TKeyExtractor = TIdentity, class TPredicate = TLess<TKeyType>, class A = std::allocator<TValueType>> - class TSimpleSet: public TSortedVector<TValueType, TKeyType, TKeyExtractor, TPredicate, A> { - private: - typedef TSortedVector<TValueType, TKeyType, TKeyExtractor, TPredicate, A> TBase; - - public: - typedef typename TBase::value_type value_type; - typedef typename TBase::key_type key_type; - typedef typename TBase::iterator iterator; - typedef typename TBase::const_iterator const_iterator; - typedef typename TBase::size_type size_type; + class TSimpleSet: public TSortedVector<TValueType, TKeyType, TKeyExtractor, TPredicate, A> { + private: + typedef TSortedVector<TValueType, TKeyType, TKeyExtractor, TPredicate, A> TBase; + + public: + typedef typename TBase::value_type value_type; + typedef typename TBase::key_type key_type; + typedef typename TBase::iterator iterator; + typedef typename TBase::const_iterator const_iterator; + typedef typename TBase::size_type size_type; typedef NPrivate::TEqual<TPredicate> TKeyEqual; - - public: - inline TSimpleSet() - : TBase() - { - } - - template <class TIter> - inline TSimpleSet(TIter first, TIter last) - : TBase(first, last) - { - TBase::MakeUnique(); - } - + + public: + inline TSimpleSet() + : TBase() + { + } + + template <class TIter> + inline TSimpleSet(TIter first, TIter last) + : TBase(first, last) + { + TBase::MakeUnique(); + } + inline TSimpleSet(std::initializer_list<value_type> il) : TBase(il) { TBase::MakeUnique(); } - // The method expects that there is a TValueType(TKeyType) constructor available - inline TValueType& Get(const TKeyType& key) { - typename TBase::iterator i = TBase::LowerBound(key); - if (i == TBase::end() || !TKeyEqual()(TKeyExtractor()(*i), key)) + // The method expects that there is a TValueType(TKeyType) constructor available + inline TValueType& Get(const TKeyType& key) { + typename TBase::iterator i = TBase::LowerBound(key); + if (i == TBase::end() || !TKeyEqual()(TKeyExtractor()(*i), key)) i = TVector<TValueType, A>::insert(i, TValueType(key)); - return *i; - } - + return *i; + } + template<class K> inline const TValueType& Get(const K& key, const TValueType& def) const { - typename TBase::const_iterator i = TBase::Find(key); - return i != TBase::end() ? *i : def; - } - + typename TBase::const_iterator i = TBase::Find(key); + return i != TBase::end() ? *i : def; + } + template<class K> Y_FORCE_INLINE TValueType& operator[](const K& key) { - return Get(key); - } - - // Inserts value with unique key. Returns <iterator, bool> pair, - // where the first member is the pointer to the inserted/existing value, and the - // second member indicates either the value is inserted or not. + return Get(key); + } + + // Inserts value with unique key. Returns <iterator, bool> pair, + // where the first member is the pointer to the inserted/existing value, and the + // second member indicates either the value is inserted or not. Y_FORCE_INLINE std::pair<iterator, bool> Insert(const TValueType& value) { - return TBase::InsertUnique(value); - } - - // STL-compatible synonym + return TBase::InsertUnique(value); + } + + // STL-compatible synonym Y_FORCE_INLINE std::pair<iterator, bool> insert(const TValueType& value) { - return TBase::InsertUnique(value); - } - - // Inserts value range with unique keys. - template <class TIter> + return TBase::InsertUnique(value); + } + + // Inserts value range with unique keys. + template <class TIter> Y_FORCE_INLINE void Insert(TIter first, TIter last) { - TBase::InsertUnique(first, last); - } - - // STL-compatible synonym - template <class TIter> + TBase::InsertUnique(first, last); + } + + // STL-compatible synonym + template <class TIter> Y_FORCE_INLINE void insert(TIter first, TIter last) { - TBase::InsertUnique(first, last); - } - }; - -} - -template <typename V, typename K, class E, class P, class A> -class TSerializer<NSorted::TSortedVector<V, K, E, P, A>>: public TVectorSerializer<NSorted::TSortedVector<V, K, E, P, A>> { -}; - -template <typename K, typename V, class P, class A> -class TSerializer<NSorted::TSimpleMap<K, V, P, A>>: public TVectorSerializer<NSorted::TSimpleMap<K, V, P, A>> { -}; - -template <typename V, typename K, class E, class P, class A> -class TSerializer<NSorted::TSimpleSet<V, K, E, P, A>>: public TVectorSerializer<NSorted::TSimpleSet<V, K, E, P, A>> { -}; + TBase::InsertUnique(first, last); + } + }; + +} + +template <typename V, typename K, class E, class P, class A> +class TSerializer<NSorted::TSortedVector<V, K, E, P, A>>: public TVectorSerializer<NSorted::TSortedVector<V, K, E, P, A>> { +}; + +template <typename K, typename V, class P, class A> +class TSerializer<NSorted::TSimpleMap<K, V, P, A>>: public TVectorSerializer<NSorted::TSimpleMap<K, V, P, A>> { +}; + +template <typename V, typename K, class E, class P, class A> +class TSerializer<NSorted::TSimpleSet<V, K, E, P, A>>: public TVectorSerializer<NSorted::TSimpleSet<V, K, E, P, A>> { +}; diff --git a/library/cpp/containers/sorted_vector/ya.make b/library/cpp/containers/sorted_vector/ya.make index 1975c5dc90..78f1c1494e 100644 --- a/library/cpp/containers/sorted_vector/ya.make +++ b/library/cpp/containers/sorted_vector/ya.make @@ -1,11 +1,11 @@ -LIBRARY() - -OWNER(udovichenko-r) - -SRCS( - sorted_vector.cpp -) - -END() +LIBRARY() + +OWNER(udovichenko-r) + +SRCS( + sorted_vector.cpp +) + +END() RECURSE_FOR_TESTS(ut) diff --git a/library/cpp/containers/stack_array/range_ops.cpp b/library/cpp/containers/stack_array/range_ops.cpp index f1b5e3af0d..ea47fe6777 100644 --- a/library/cpp/containers/stack_array/range_ops.cpp +++ b/library/cpp/containers/stack_array/range_ops.cpp @@ -1 +1 @@ -#include "range_ops.h" +#include "range_ops.h" diff --git a/library/cpp/containers/stack_array/range_ops.h b/library/cpp/containers/stack_array/range_ops.h index 1d40341aa1..dfb95ab401 100644 --- a/library/cpp/containers/stack_array/range_ops.h +++ b/library/cpp/containers/stack_array/range_ops.h @@ -1,52 +1,52 @@ -#pragma once - -#include <util/generic/typetraits.h> - -#include <new> - -namespace NRangeOps { - template <class T, bool isTrivial> - struct TRangeOpsBase { +#pragma once + +#include <util/generic/typetraits.h> + +#include <new> + +namespace NRangeOps { + template <class T, bool isTrivial> + struct TRangeOpsBase { static inline void DestroyRange(T* b, T* e) noexcept { - while (e > b) { - (--e)->~T(); - } - } - - static inline void InitializeRange(T* b, T* e) { - T* c = b; - - try { - for (; c < e; ++c) { - new (c) T(); - } - } catch (...) { - DestroyRange(b, c); - - throw; - } - } - }; - - template <class T> - struct TRangeOpsBase<T, true> { + while (e > b) { + (--e)->~T(); + } + } + + static inline void InitializeRange(T* b, T* e) { + T* c = b; + + try { + for (; c < e; ++c) { + new (c) T(); + } + } catch (...) { + DestroyRange(b, c); + + throw; + } + } + }; + + template <class T> + struct TRangeOpsBase<T, true> { static inline void DestroyRange(T*, T*) noexcept { - } - + } + static inline void InitializeRange(T*, T*) noexcept { - } - }; - - template <class T> - using TRangeOps = TRangeOpsBase<T, TTypeTraits<T>::IsPod>; - - template <class T> + } + }; + + template <class T> + using TRangeOps = TRangeOpsBase<T, TTypeTraits<T>::IsPod>; + + template <class T> static inline void DestroyRange(T* b, T* e) noexcept { - TRangeOps<T>::DestroyRange(b, e); - } - - template <class T> - static inline void InitializeRange(T* b, T* e) { - TRangeOps<T>::InitializeRange(b, e); - } -} + TRangeOps<T>::DestroyRange(b, e); + } + + template <class T> + static inline void InitializeRange(T* b, T* e) { + TRangeOps<T>::InitializeRange(b, e); + } +} diff --git a/library/cpp/containers/stack_array/stack_array.cpp b/library/cpp/containers/stack_array/stack_array.cpp index 68e8b097ba..d579328128 100644 --- a/library/cpp/containers/stack_array/stack_array.cpp +++ b/library/cpp/containers/stack_array/stack_array.cpp @@ -1 +1 @@ -#include "stack_array.h" +#include "stack_array.h" diff --git a/library/cpp/containers/stack_array/stack_array.h b/library/cpp/containers/stack_array/stack_array.h index 28e49bfc3c..105189de06 100644 --- a/library/cpp/containers/stack_array/stack_array.h +++ b/library/cpp/containers/stack_array/stack_array.h @@ -1,11 +1,11 @@ -#pragma once - -#include "range_ops.h" - +#pragma once + +#include "range_ops.h" + #include <util/generic/array_ref.h> #include <util/system/defaults.h> /* For alloca. */ - -namespace NStackArray { + +namespace NStackArray { /** * A stack-allocated array. Should be used instead of � variable length * arrays that are not part of C++ standard. @@ -21,20 +21,20 @@ namespace NStackArray { * * Note that it is generally a *VERY BAD* idea to use this in inline methods * as those might be called from a loop, and then stack overflow is in the cards. - */ - template <class T> + */ + template <class T> class TStackArray: public TArrayRef<T> { - public: - inline TStackArray(void* data, size_t len) + public: + inline TStackArray(void* data, size_t len) : TArrayRef<T>((T*)data, len) - { + { NRangeOps::InitializeRange(this->begin(), this->end()); - } - + } + inline ~TStackArray() { NRangeOps::DestroyRange(this->begin(), this->end()); - } - }; -} - -#define ALLOC_ON_STACK(type, n) alloca(sizeof(type) * (n)), (n) + } + }; +} + +#define ALLOC_ON_STACK(type, n) alloca(sizeof(type) * (n)), (n) diff --git a/library/cpp/containers/stack_array/ut/tests_ut.cpp b/library/cpp/containers/stack_array/ut/tests_ut.cpp index 3e96384f0e..d0c4bfe8c0 100644 --- a/library/cpp/containers/stack_array/ut/tests_ut.cpp +++ b/library/cpp/containers/stack_array/ut/tests_ut.cpp @@ -1,94 +1,94 @@ #include <library/cpp/containers/stack_array/stack_array.h> #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TestStackArray) { - using namespace NStackArray; - - static inline void* FillWithTrash(void* d, size_t l) { - memset(d, 0xCC, l); - - return d; - } - -#define ALLOC(type, len) FillWithTrash(alloca(sizeof(type) * len), sizeof(type) * len), len - + using namespace NStackArray; + + static inline void* FillWithTrash(void* d, size_t l) { + memset(d, 0xCC, l); + + return d; + } + +#define ALLOC(type, len) FillWithTrash(alloca(sizeof(type) * len), sizeof(type) * len), len + Y_UNIT_TEST(Test1) { - TStackArray<ui32> s(ALLOC(ui32, 10)); - + TStackArray<ui32> s(ALLOC(ui32, 10)); + UNIT_ASSERT_VALUES_EQUAL(s.size(), 10); - + for (size_t i = 0; i < s.size(); ++i) { - UNIT_ASSERT_VALUES_EQUAL(s[i], 0xCCCCCCCC); - } - - for (auto&& x : s) { - UNIT_ASSERT_VALUES_EQUAL(x, 0xCCCCCCCC); - } - + UNIT_ASSERT_VALUES_EQUAL(s[i], 0xCCCCCCCC); + } + + for (auto&& x : s) { + UNIT_ASSERT_VALUES_EQUAL(x, 0xCCCCCCCC); + } + for (size_t i = 0; i < s.size(); ++i) { - s[i] = i; - } - - size_t ss = 0; - - for (auto&& x : s) { - ss += x; - } - - UNIT_ASSERT_VALUES_EQUAL(ss, 45); - } - - static int N1 = 0; - - struct TX1 { - inline TX1() { - ++N1; - } - - inline ~TX1() { - --N1; - } - }; - + s[i] = i; + } + + size_t ss = 0; + + for (auto&& x : s) { + ss += x; + } + + UNIT_ASSERT_VALUES_EQUAL(ss, 45); + } + + static int N1 = 0; + + struct TX1 { + inline TX1() { + ++N1; + } + + inline ~TX1() { + --N1; + } + }; + Y_UNIT_TEST(Test2) { - { - TStackArray<TX1> s(ALLOC(TX1, 10)); - - UNIT_ASSERT_VALUES_EQUAL(N1, 10); - } - - UNIT_ASSERT_VALUES_EQUAL(N1, 0); - } - - static int N2 = 0; - static int N3 = 0; - - struct TX2 { - inline TX2() { - if (N2 >= 5) { - ythrow yexception() << "ups"; - } - - ++N3; - ++N2; - } - - inline ~TX2() { - --N2; - } - }; - + { + TStackArray<TX1> s(ALLOC(TX1, 10)); + + UNIT_ASSERT_VALUES_EQUAL(N1, 10); + } + + UNIT_ASSERT_VALUES_EQUAL(N1, 0); + } + + static int N2 = 0; + static int N3 = 0; + + struct TX2 { + inline TX2() { + if (N2 >= 5) { + ythrow yexception() << "ups"; + } + + ++N3; + ++N2; + } + + inline ~TX2() { + --N2; + } + }; + Y_UNIT_TEST(Test3) { - bool haveException = false; - - try { - TStackArray<TX2> s(ALLOC_ON_STACK(TX2, 10)); - } catch (...) { - haveException = true; - } - - UNIT_ASSERT(haveException); - UNIT_ASSERT_VALUES_EQUAL(N2, 0); - UNIT_ASSERT_VALUES_EQUAL(N3, 5); - } -} + bool haveException = false; + + try { + TStackArray<TX2> s(ALLOC_ON_STACK(TX2, 10)); + } catch (...) { + haveException = true; + } + + UNIT_ASSERT(haveException); + UNIT_ASSERT_VALUES_EQUAL(N2, 0); + UNIT_ASSERT_VALUES_EQUAL(N3, 5); + } +} diff --git a/library/cpp/containers/stack_array/ut/ya.make b/library/cpp/containers/stack_array/ut/ya.make index 7db7340073..f224ff2942 100644 --- a/library/cpp/containers/stack_array/ut/ya.make +++ b/library/cpp/containers/stack_array/ut/ya.make @@ -1,9 +1,9 @@ UNITTEST_FOR(library/cpp/containers/stack_array) - -OWNER(pg) - -SRCS( - tests_ut.cpp -) - -END() + +OWNER(pg) + +SRCS( + tests_ut.cpp +) + +END() diff --git a/library/cpp/containers/stack_array/ya.make b/library/cpp/containers/stack_array/ya.make index 9bc0afc66c..ad84393f02 100644 --- a/library/cpp/containers/stack_array/ya.make +++ b/library/cpp/containers/stack_array/ya.make @@ -1,10 +1,10 @@ -LIBRARY() - -OWNER(pg) - -SRCS( - range_ops.cpp - stack_array.cpp -) - -END() +LIBRARY() + +OWNER(pg) + +SRCS( + range_ops.cpp + stack_array.cpp +) + +END() diff --git a/library/cpp/containers/stack_vector/stack_vec.h b/library/cpp/containers/stack_vector/stack_vec.h index fcc5d9a2a5..0108642da1 100644 --- a/library/cpp/containers/stack_vector/stack_vec.h +++ b/library/cpp/containers/stack_vector/stack_vec.h @@ -20,27 +20,27 @@ using TSmallVec = TStackVec<T, 16, true, Alloc>; template <typename T, size_t CountOnStack = 256> using TStackOnlyVec = TStackVec<T, CountOnStack, false>; -namespace NPrivate { - template <class Alloc, class StackAlloc, typename T, typename U> - struct TRebind { +namespace NPrivate { + template <class Alloc, class StackAlloc, typename T, typename U> + struct TRebind { typedef TReboundAllocator<Alloc, U> other; }; - template <class Alloc, class StackAlloc, typename T> - struct TRebind<Alloc, StackAlloc, T, T> { + template <class Alloc, class StackAlloc, typename T> + struct TRebind<Alloc, StackAlloc, T, T> { typedef StackAlloc other; }; template <typename T, size_t CountOnStack, bool UseFallbackAlloc, class Alloc = std::allocator<T>> - class TStackBasedAllocator: public Alloc { + class TStackBasedAllocator: public Alloc { public: typedef TStackBasedAllocator<T, CountOnStack, UseFallbackAlloc, Alloc> TSelf; - using typename Alloc::difference_type; + using typename Alloc::difference_type; using typename Alloc::size_type; - using typename Alloc::value_type; + using typename Alloc::value_type; - template <class U> + template <class U> struct rebind: public ::NPrivate::TRebind<Alloc, TSelf, T, U> { }; @@ -89,7 +89,7 @@ namespace NPrivate { return static_cast<Alloc&>(*this); } }; -} +} template <typename T, size_t CountOnStack, bool UseFallbackAlloc, class Alloc> class TStackVec: public TVector<T, ::NPrivate::TStackBasedAllocator<T, CountOnStack, UseFallbackAlloc, TReboundAllocator<Alloc, T>>> { @@ -98,11 +98,11 @@ private: using TAllocator = typename TBase::allocator_type; public: - using typename TBase::const_iterator; - using typename TBase::const_reverse_iterator; + using typename TBase::const_iterator; + using typename TBase::const_reverse_iterator; using typename TBase::iterator; using typename TBase::reverse_iterator; - using typename TBase::size_type; + using typename TBase::size_type; using typename TBase::value_type; public: diff --git a/library/cpp/containers/str_map/str_map.cpp b/library/cpp/containers/str_map/str_map.cpp index 58c65babda..1b65718d42 100644 --- a/library/cpp/containers/str_map/str_map.cpp +++ b/library/cpp/containers/str_map/str_map.cpp @@ -1 +1 @@ -#include "str_map.h" +#include "str_map.h" diff --git a/library/cpp/containers/str_map/str_map.h b/library/cpp/containers/str_map/str_map.h index 31b00d1b99..47a4540787 100644 --- a/library/cpp/containers/str_map/str_map.h +++ b/library/cpp/containers/str_map/str_map.h @@ -1,44 +1,44 @@ #pragma once -#include <util/memory/segmented_string_pool.h> +#include <util/memory/segmented_string_pool.h> #include <util/generic/map.h> #include <util/generic/hash.h> -#include <util/generic/buffer.h> -#include <util/str_stl.h> // less<> and equal_to<> for const char* +#include <util/generic/buffer.h> +#include <util/str_stl.h> // less<> and equal_to<> for const char* #include <utility> #include <util/generic/noncopyable.h> -template <class T, class HashFcn = THash<const char*>, class EqualTo = TEqualTo<const char*>, class Alloc = std::allocator<const char*>> -class string_hash; - -template <class T, class HashFcn = THash<const char*>, class EqualTo = TEqualTo<const char*>> -class segmented_string_hash; - +template <class T, class HashFcn = THash<const char*>, class EqualTo = TEqualTo<const char*>, class Alloc = std::allocator<const char*>> +class string_hash; + +template <class T, class HashFcn = THash<const char*>, class EqualTo = TEqualTo<const char*>> +class segmented_string_hash; + template <class Map> inline std::pair<typename Map::iterator, bool> -pool_insert(Map* m, const char* key, const typename Map::mapped_type& data, TBuffer& pool) { +pool_insert(Map* m, const char* key, const typename Map::mapped_type& data, TBuffer& pool) { std::pair<typename Map::iterator, bool> ins = m->insert(typename Map::value_type(key, data)); - if (ins.second) { // new? - size_t buflen = strlen(key) + 1; // strlen??? - const char* old_pool = pool.Begin(); - pool.Append(key, buflen); - if (pool.Begin() != old_pool) // repoint? + if (ins.second) { // new? + size_t buflen = strlen(key) + 1; // strlen??? + const char* old_pool = pool.Begin(); + pool.Append(key, buflen); + if (pool.Begin() != old_pool) // repoint? for (typename Map::iterator it = m->begin(); it != m->end(); ++it) if ((*it).first != key) - const_cast<const char*&>((*it).first) += (pool.Begin() - old_pool); - const_cast<const char*&>((*ins.first).first) = pool.End() - buflen; + const_cast<const char*&>((*it).first) += (pool.Begin() - old_pool); + const_cast<const char*&>((*ins.first).first) = pool.End() - buflen; } return ins; } #define HASH_SIZE_DEFAULT 100 -#define AVERAGEWORD_BUF 10 +#define AVERAGEWORD_BUF 10 template <class T, class HashFcn, class EqualTo, class Alloc> class string_hash: public THashMap<const char*, T, HashFcn, EqualTo, Alloc> { protected: - TBuffer pool; - + TBuffer pool; + public: using yh = THashMap<const char*, T, HashFcn, EqualTo, Alloc>; using iterator = typename yh::iterator; @@ -46,25 +46,25 @@ public: using mapped_type = typename yh::mapped_type; using size_type = typename yh::size_type; using pool_size_type = typename yh::size_type; - string_hash() { - pool.Reserve(HASH_SIZE_DEFAULT * AVERAGEWORD_BUF); // reserve here + string_hash() { + pool.Reserve(HASH_SIZE_DEFAULT * AVERAGEWORD_BUF); // reserve here } string_hash(size_type hash_size, pool_size_type pool_size) : THashMap<const char*, T, HashFcn, EqualTo, Alloc>(hash_size) { - pool.Reserve(pool_size); // reserve here + pool.Reserve(pool_size); // reserve here } std::pair<iterator, bool> insert_copy(const char* key, const mapped_type& data) { return ::pool_insert(this, key, data, pool); } - void clear_hash() { + void clear_hash() { yh::clear(); - pool.Clear(); + pool.Clear(); } - pool_size_type pool_size() const { - return pool.Size(); + pool_size_type pool_size() const { + return pool.Size(); } string_hash(const string_hash& sh) @@ -83,7 +83,7 @@ public: (const char*&)(*i).first += delta; } */ - string_hash& operator=(const string_hash& sh) { + string_hash& operator=(const string_hash& sh) { if (&sh != this) { clear_hash(); for (const_iterator i = sh.begin(); i != sh.end(); ++i) @@ -92,7 +92,7 @@ public: return *this; } - mapped_type& operator[](const char* key) { + mapped_type& operator[](const char* key) { iterator I = yh::find(key); if (I == yh::end()) I = insert_copy(key, mapped_type()).first; @@ -104,7 +104,7 @@ template <class C, class T, class HashFcn, class EqualTo> class THashWithSegmentedPoolForKeys: protected THashMap<const C*, T, HashFcn, EqualTo>, TNonCopyable { protected: segmented_pool<C> pool; - + public: using yh = THashMap<const C*, T, HashFcn, EqualTo>; using iterator = typename yh::iterator; @@ -176,7 +176,7 @@ public: }; template <class T, class HashFcn, class EqualTo> -class segmented_string_hash: public THashWithSegmentedPoolForKeys<char, T, HashFcn, EqualTo> { +class segmented_string_hash: public THashWithSegmentedPoolForKeys<char, T, HashFcn, EqualTo> { public: using Base = THashWithSegmentedPoolForKeys<char, T, HashFcn, EqualTo>; using iterator = typename Base::iterator; @@ -185,10 +185,10 @@ public: using size_type = typename Base::size_type; using key_type = typename Base::key_type; using value_type = typename Base::value_type; - + public: segmented_string_hash(size_type hash_size = HASH_SIZE_DEFAULT, size_t segsize = HASH_SIZE_DEFAULT * AVERAGEWORD_BUF, bool afs = false) - : Base(hash_size, segsize, afs) + : Base(hash_size, segsize, afs) { } diff --git a/library/cpp/containers/str_map/ya.make b/library/cpp/containers/str_map/ya.make index b834159cda..9c887fd9e2 100644 --- a/library/cpp/containers/str_map/ya.make +++ b/library/cpp/containers/str_map/ya.make @@ -1,9 +1,9 @@ -LIBRARY() - -OWNER(pg) - -SRCS( - str_map.cpp -) - -END() +LIBRARY() + +OWNER(pg) + +SRCS( + str_map.cpp +) + +END() diff --git a/library/cpp/containers/top_keeper/top_keeper.h b/library/cpp/containers/top_keeper/top_keeper.h index 2f282b5a9e..b5f901a86e 100644 --- a/library/cpp/containers/top_keeper/top_keeper.h +++ b/library/cpp/containers/top_keeper/top_keeper.h @@ -14,13 +14,13 @@ private: size_t HalfMaxSize; TComparator Comparer; size_t MinElementIndex; - + private: void Reserve() { Internal.reserve(2 * HalfMaxSize); } - template <class UT> + template <class UT> bool Insert(UT&& value) noexcept { if (Y_UNLIKELY(0 == HalfMaxSize)) { return false; @@ -55,7 +55,7 @@ private: Reserve(); } - template <class TAllocParam> + template <class TAllocParam> TVectorWithMin(const size_t halfMaxSize, const TComparator& comp, TAllocParam&& param) : Internal(std::forward<TAllocParam>(param)) , HalfMaxSize(halfMaxSize) @@ -89,7 +89,7 @@ private: return Insert(std::move(value)); } - template <class... TArgs> + template <class... TArgs> bool Emplace(TArgs&&... args) { return Insert(T(std::forward<TArgs>(args)...)); // TODO: make it "real" emplace, not that fake one } @@ -133,7 +133,7 @@ private: void CheckNotFinalized() { Y_ENSURE(!Finalized, "Cannot insert after finalizing (Pop() / GetNext() / Finalize())! " - "Use TLimitedHeap for this scenario"); + "Use TLimitedHeap for this scenario"); } size_t MaxSize; @@ -158,7 +158,7 @@ public: { } - template <class TAllocParam> + template <class TAllocParam> TTopKeeper(size_t maxSize, const TComparator& comp, TAllocParam&& param) : MaxSize(maxSize) , Comparer(comp) @@ -219,7 +219,7 @@ public: return Internal.Push(std::move(value)); } - template <class... TArgs> + template <class... TArgs> bool Emplace(TArgs&&... args) { CheckNotFinalized(); return Internal.Emplace(std::forward<TArgs>(args)...); @@ -244,7 +244,7 @@ public: void SetMaxSize(size_t newMaxSize) { Y_ENSURE(!Finalized, "Cannot resize after finalizing (Pop() / GetNext() / Finalize())! " - "Use TLimitedHeap for this scenario"); + "Use TLimitedHeap for this scenario"); MaxSize = newMaxSize; Internal.SetMaxSize(newMaxSize); } diff --git a/library/cpp/containers/top_keeper/top_keeper/top_keeper.h b/library/cpp/containers/top_keeper/top_keeper/top_keeper.h index 2f282b5a9e..b5f901a86e 100644 --- a/library/cpp/containers/top_keeper/top_keeper/top_keeper.h +++ b/library/cpp/containers/top_keeper/top_keeper/top_keeper.h @@ -14,13 +14,13 @@ private: size_t HalfMaxSize; TComparator Comparer; size_t MinElementIndex; - + private: void Reserve() { Internal.reserve(2 * HalfMaxSize); } - template <class UT> + template <class UT> bool Insert(UT&& value) noexcept { if (Y_UNLIKELY(0 == HalfMaxSize)) { return false; @@ -55,7 +55,7 @@ private: Reserve(); } - template <class TAllocParam> + template <class TAllocParam> TVectorWithMin(const size_t halfMaxSize, const TComparator& comp, TAllocParam&& param) : Internal(std::forward<TAllocParam>(param)) , HalfMaxSize(halfMaxSize) @@ -89,7 +89,7 @@ private: return Insert(std::move(value)); } - template <class... TArgs> + template <class... TArgs> bool Emplace(TArgs&&... args) { return Insert(T(std::forward<TArgs>(args)...)); // TODO: make it "real" emplace, not that fake one } @@ -133,7 +133,7 @@ private: void CheckNotFinalized() { Y_ENSURE(!Finalized, "Cannot insert after finalizing (Pop() / GetNext() / Finalize())! " - "Use TLimitedHeap for this scenario"); + "Use TLimitedHeap for this scenario"); } size_t MaxSize; @@ -158,7 +158,7 @@ public: { } - template <class TAllocParam> + template <class TAllocParam> TTopKeeper(size_t maxSize, const TComparator& comp, TAllocParam&& param) : MaxSize(maxSize) , Comparer(comp) @@ -219,7 +219,7 @@ public: return Internal.Push(std::move(value)); } - template <class... TArgs> + template <class... TArgs> bool Emplace(TArgs&&... args) { CheckNotFinalized(); return Internal.Emplace(std::forward<TArgs>(args)...); @@ -244,7 +244,7 @@ public: void SetMaxSize(size_t newMaxSize) { Y_ENSURE(!Finalized, "Cannot resize after finalizing (Pop() / GetNext() / Finalize())! " - "Use TLimitedHeap for this scenario"); + "Use TLimitedHeap for this scenario"); MaxSize = newMaxSize; Internal.SetMaxSize(newMaxSize); } diff --git a/library/cpp/coroutine/engine/condvar.h b/library/cpp/coroutine/engine/condvar.h index ffceede6fa..384a6c19f8 100644 --- a/library/cpp/coroutine/engine/condvar.h +++ b/library/cpp/coroutine/engine/condvar.h @@ -1,38 +1,38 @@ #pragma once - + #include "events.h" #include "mutex.h" -class TContCondVar { -public: +class TContCondVar { +public: int WaitD(TCont* current, TContMutex* mutex, TInstant deadline) { - mutex->UnLock(); - - const int ret = WaitQueue_.WaitD(current, deadline); - - if (ret != EWAKEDUP) { + mutex->UnLock(); + + const int ret = WaitQueue_.WaitD(current, deadline); + + if (ret != EWAKEDUP) { return ret; } - return mutex->LockD(current, deadline); - } - + return mutex->LockD(current, deadline); + } + int WaitT(TCont* current, TContMutex* mutex, TDuration timeout) { - return WaitD(current, mutex, timeout.ToDeadLine()); - } - + return WaitD(current, mutex, timeout.ToDeadLine()); + } + int WaitI(TCont* current, TContMutex* mutex) { - return WaitD(current, mutex, TInstant::Max()); - } - + return WaitD(current, mutex, TInstant::Max()); + } + void Signal() noexcept { - WaitQueue_.Signal(); - } - + WaitQueue_.Signal(); + } + void BroadCast() noexcept { - WaitQueue_.BroadCast(); - } - -private: - TContWaitQueue WaitQueue_; -}; + WaitQueue_.BroadCast(); + } + +private: + TContWaitQueue WaitQueue_; +}; diff --git a/library/cpp/coroutine/engine/cont_poller.cpp b/library/cpp/coroutine/engine/cont_poller.cpp index 76593d4e9b..6073b72e62 100644 --- a/library/cpp/coroutine/engine/cont_poller.cpp +++ b/library/cpp/coroutine/engine/cont_poller.cpp @@ -1,6 +1,6 @@ #include "cont_poller.h" -#include "impl.h" - +#include "impl.h" + namespace NCoro { namespace { template <class T> @@ -20,8 +20,8 @@ namespace NCoro { return event->Status(); } - } - + } + void TContPollEvent::Wake() noexcept { UnLink(); Cont()->ReSchedule(); @@ -55,7 +55,7 @@ namespace NCoro { }; IoWait_.ForEach(visitor); } -} +} void TFdEvent::RemoveFromIOWait() noexcept { this->Cont()->Executor()->Poller()->Remove(this); diff --git a/library/cpp/coroutine/engine/cont_poller.h b/library/cpp/coroutine/engine/cont_poller.h index b638b2df1a..e6b23149c3 100644 --- a/library/cpp/coroutine/engine/cont_poller.h +++ b/library/cpp/coroutine/engine/cont_poller.h @@ -1,27 +1,27 @@ #pragma once - -#include "poller.h" + +#include "poller.h" #include "sockmap.h" - + #include <library/cpp/containers/intrusive_rb_tree/rb_tree.h> #include <util/datetime/base.h> #include <util/memory/pool.h> -#include <util/memory/smallobj.h> +#include <util/memory/smallobj.h> #include <util/network/init.h> - + #include <cerrno> -class TCont; -class TContExecutor; +class TCont; +class TContExecutor; class TFdEvent; - + namespace NCoro { class IPollEvent; - - + + struct TContPollEventCompare { template <class T> static inline bool Compare(const T& l, const T& r) noexcept { @@ -29,74 +29,74 @@ namespace NCoro { } }; - + class TContPollEvent : public TRbTreeItem<TContPollEvent, TContPollEventCompare> { public: TContPollEvent(TCont* cont, TInstant deadLine) noexcept : Cont_(cont) , DeadLine_(deadLine) {} - + static bool Compare(const TContPollEvent& l, const TContPollEvent& r) noexcept { return l.DeadLine() < r.DeadLine() || (l.DeadLine() == r.DeadLine() && &l < &r); } - + int Status() const noexcept { return Status_; } - + void SetStatus(int status) noexcept { Status_ = status; } - + TCont* Cont() noexcept { return Cont_; } - + TInstant DeadLine() const noexcept { return DeadLine_; } - + void Wake(int status) noexcept { SetStatus(status); Wake(); - } - + } + private: void Wake() noexcept; - + private: TCont* Cont_; TInstant DeadLine_; int Status_ = EINPROGRESS; - }; - - + }; + + class IPollEvent: public TIntrusiveListItem<IPollEvent> { public: IPollEvent(SOCKET fd, ui16 what) noexcept : Fd_(fd) , What_(what) {} - + virtual ~IPollEvent() {} - + SOCKET Fd() const noexcept { return Fd_; } - + int What() const noexcept { return What_; } - + virtual void OnPollEvent(int status) noexcept = 0; - + private: SOCKET Fd_; ui16 What_; }; - - + + template <class T> class TBigArray { struct TValue: public T, public TObjectFromPool<TValue> { @@ -124,27 +124,27 @@ namespace NCoro { using TPollEventList = TIntrusiveList<IPollEvent>; - + class TContPoller { public: using TEvent = IPollerFace::TEvent; using TEvents = IPollerFace::TEvents; - + TContPoller() : P_(IPollerFace::Default()) { } - + explicit TContPoller(THolder<IPollerFace> poller) : P_(std::move(poller)) {} - + void Schedule(IPollEvent* event) { auto* lst = Lists_.Get(event->Fd()); const ui16 oldFlags = Flags(*lst); lst->PushFront(event); ui16 newFlags = Flags(*lst); - + if (newFlags != oldFlags) { if (oldFlags) { newFlags |= CONT_POLL_MODIFY; @@ -152,14 +152,14 @@ namespace NCoro { P_->Set(lst, event->Fd(), newFlags); } - } - + } + void Remove(IPollEvent* event) noexcept { auto* lst = Lists_.Get(event->Fd()); const ui16 oldFlags = Flags(*lst); event->Unlink(); ui16 newFlags = Flags(*lst); - + if (newFlags != oldFlags) { if (newFlags) { newFlags |= CONT_POLL_MODIFY; @@ -167,13 +167,13 @@ namespace NCoro { P_->Set(lst, event->Fd(), newFlags); } - } - + } + void Wait(TEvents& events, TInstant deadLine) { events.clear(); P_->Wait(events, deadLine); - } - + } + EContPoller PollEngine() const { return P_->PollEngine(); } @@ -182,14 +182,14 @@ namespace NCoro { ui16 ret = 0; for (auto&& item : lst) { ret |= item.What(); - } + } return ret; - } - + } + private: TBigArray<TPollEventList> Lists_; THolder<IPollerFace> P_; - }; + }; class TEventWaitQueue { @@ -210,12 +210,12 @@ namespace NCoro { TIoWait IoWait_; }; } - + class TFdEvent final: public NCoro::TContPollEvent, public NCoro::IPollEvent { -public: +public: TFdEvent(TCont* cont, SOCKET fd, ui16 what, TInstant deadLine) noexcept : TContPollEvent(cont, deadLine) , IPollEvent(fd, what) @@ -223,22 +223,22 @@ public: ~TFdEvent() { RemoveFromIOWait(); - } - + } + void RemoveFromIOWait() noexcept; - + void OnPollEvent(int status) noexcept override { Wake(status); - } -}; - + } +}; + class TTimerEvent: public NCoro::TContPollEvent { -public: +public: TTimerEvent(TCont* cont, TInstant deadLine) noexcept : TContPollEvent(cont, deadLine) {} -}; +}; int ExecuteEvent(TFdEvent* event) noexcept; diff --git a/library/cpp/coroutine/engine/coroutine_ut.cpp b/library/cpp/coroutine/engine/coroutine_ut.cpp index 8b372496a2..29e0d1dfc5 100644 --- a/library/cpp/coroutine/engine/coroutine_ut.cpp +++ b/library/cpp/coroutine/engine/coroutine_ut.cpp @@ -1,34 +1,34 @@ -#include "impl.h" +#include "impl.h" #include "condvar.h" #include "network.h" - + #include <library/cpp/testing/unittest/registar.h> - -#include <util/string/cast.h> + +#include <util/string/cast.h> #include <util/system/pipe.h> #include <util/system/env.h> #include <util/system/info.h> #include <util/system/thread.h> #include <util/generic/xrange.h> #include <util/generic/serialized_enum.h> - + // TODO (velavokr): BALANCER-1345 add more tests on pollers -class TCoroTest: public TTestBase { - UNIT_TEST_SUITE(TCoroTest); - UNIT_TEST(TestSimpleX1); +class TCoroTest: public TTestBase { + UNIT_TEST_SUITE(TCoroTest); + UNIT_TEST(TestSimpleX1); UNIT_TEST(TestSimpleX1MultiThread); - UNIT_TEST(TestSimpleX2); - UNIT_TEST(TestSimpleX3); - UNIT_TEST(TestMemFun); - UNIT_TEST(TestMutex); - UNIT_TEST(TestCondVar); + UNIT_TEST(TestSimpleX2); + UNIT_TEST(TestSimpleX3); + UNIT_TEST(TestMemFun); + UNIT_TEST(TestMutex); + UNIT_TEST(TestCondVar); UNIT_TEST(TestJoinDefault); UNIT_TEST(TestJoinEpoll); UNIT_TEST(TestJoinKqueue); UNIT_TEST(TestJoinPoll); UNIT_TEST(TestJoinSelect); - UNIT_TEST(TestException); + UNIT_TEST(TestException); UNIT_TEST(TestJoinCancelExitRaceBug); UNIT_TEST(TestWaitWakeLivelockBug); UNIT_TEST(TestFastPathWakeDefault) @@ -46,17 +46,17 @@ class TCoroTest: public TTestBase { UNIT_TEST(TestUserEvent); UNIT_TEST(TestPause); UNIT_TEST(TestOverrideTime); - UNIT_TEST_SUITE_END(); - -public: - void TestException(); - void TestSimpleX1(); + UNIT_TEST_SUITE_END(); + +public: + void TestException(); + void TestSimpleX1(); void TestSimpleX1MultiThread(); - void TestSimpleX2(); - void TestSimpleX3(); - void TestMemFun(); - void TestMutex(); - void TestCondVar(); + void TestSimpleX2(); + void TestSimpleX3(); + void TestMemFun(); + void TestMutex(); + void TestCondVar(); void TestJoinDefault(); void TestJoinEpoll(); void TestJoinKqueue(); @@ -78,72 +78,72 @@ public: void TestUserEvent(); void TestPause(); void TestOverrideTime(); -}; - -void TCoroTest::TestException() { - TContExecutor e(1000000); - - bool f2run = false; - - auto f1 = [&f2run](TCont* c) { - struct TCtx { - ~TCtx() { +}; + +void TCoroTest::TestException() { + TContExecutor e(1000000); + + bool f2run = false; + + auto f1 = [&f2run](TCont* c) { + struct TCtx { + ~TCtx() { Y_VERIFY(!*F2); - - C->Yield(); - } - - TCont* C; - bool* F2; - }; - - try { - TCtx ctx = {c, &f2run}; - - throw 1; - } catch (...) { - } - }; - - bool unc = true; - - auto f2 = [&unc, &f2run](TCont*) { - f2run = true; - unc = std::uncaught_exception(); - - // check segfault - try { - throw 2; - } catch (int) { - } - }; - - e.Create(f1, "f1"); - e.Create(f2, "f2"); - e.Execute(); - - UNIT_ASSERT(!unc); -} - + + C->Yield(); + } + + TCont* C; + bool* F2; + }; + + try { + TCtx ctx = {c, &f2run}; + + throw 1; + } catch (...) { + } + }; + + bool unc = true; + + auto f2 = [&unc, &f2run](TCont*) { + f2run = true; + unc = std::uncaught_exception(); + + // check segfault + try { + throw 2; + } catch (int) { + } + }; + + e.Create(f1, "f1"); + e.Create(f2, "f2"); + e.Execute(); + + UNIT_ASSERT(!unc); +} + static int i0; - -static void CoRun(TCont* c, void* /*run*/) { + +static void CoRun(TCont* c, void* /*run*/) { while (i0 < 100000) { ++i0; UNIT_ASSERT(RunningCont() == c); - c->Yield(); + c->Yield(); UNIT_ASSERT(RunningCont() == c); - } -} - -static void CoMain(TCont* c, void* /*arg*/) { + } +} + +static void CoMain(TCont* c, void* /*arg*/) { for (volatile size_t i2 = 0; i2 < 10; ++i2) { UNIT_ASSERT(RunningCont() == c); c->Executor()->Create(CoRun, nullptr, "run"); UNIT_ASSERT(RunningCont() == c); - } -} - + } +} + void TCoroTest::TestSimpleX1() { i0 = 0; TContExecutor e(32000); @@ -178,17 +178,17 @@ void TCoroTest::TestSimpleX1MultiThread() { UNIT_ASSERT_EQUAL(c, nThreads); } -struct TTestObject { +struct TTestObject { int i = 0; int j = 0; - + public: - void RunTask1(TCont*) { - i = 1; - } - void RunTask2(TCont*) { - j = 2; - } + void RunTask1(TCont*) { + i = 1; + } + void RunTask2(TCont*) { + j = 2; + } }; void TCoroTest::TestMemFun() { @@ -201,130 +201,130 @@ void TCoroTest::TestMemFun() { UNIT_ASSERT_EQUAL(obj.j, 2); } -void TCoroTest::TestSimpleX2() { - { +void TCoroTest::TestSimpleX2() { + { i0 = 0; - - { - TContExecutor e(32000); - e.Execute(CoMain); - } - + + { + TContExecutor e(32000); + e.Execute(CoMain); + } + UNIT_ASSERT_EQUAL(i0, 100000); - } - - { + } + + { i0 = 0; - - { - TContExecutor e(32000); - e.Execute(CoMain); - } - + + { + TContExecutor e(32000); + e.Execute(CoMain); + } + UNIT_ASSERT_EQUAL(i0, 100000); - } -} - -struct TRunner { - inline TRunner() - : Runs(0) - { - } - - inline void operator()(TCont* c) { - ++Runs; - c->Yield(); - } - - size_t Runs; -}; - -void TCoroTest::TestSimpleX3() { - TContExecutor e(32000); - TRunner runner; - + } +} + +struct TRunner { + inline TRunner() + : Runs(0) + { + } + + inline void operator()(TCont* c) { + ++Runs; + c->Yield(); + } + + size_t Runs; +}; + +void TCoroTest::TestSimpleX3() { + TContExecutor e(32000); + TRunner runner; + for (volatile size_t i3 = 0; i3 < 1000; ++i3) { - e.Create(runner, "runner"); - } - - e.Execute(); - - UNIT_ASSERT_EQUAL(runner.Runs, 1000); -} - + e.Create(runner, "runner"); + } + + e.Execute(); + + UNIT_ASSERT_EQUAL(runner.Runs, 1000); +} + static TString res; -static TContMutex mutex; - -static void CoMutex(TCont* c, void* /*run*/) { - { - mutex.LockI(c); - c->Yield(); - res += c->Name(); - mutex.UnLock(); - } - - c->Yield(); - - { - mutex.LockI(c); - c->Yield(); - res += c->Name(); - mutex.UnLock(); - } -} - -static void CoMutexTest(TCont* c, void* /*run*/) { +static TContMutex mutex; + +static void CoMutex(TCont* c, void* /*run*/) { + { + mutex.LockI(c); + c->Yield(); + res += c->Name(); + mutex.UnLock(); + } + + c->Yield(); + + { + mutex.LockI(c); + c->Yield(); + res += c->Name(); + mutex.UnLock(); + } +} + +static void CoMutexTest(TCont* c, void* /*run*/) { c->Executor()->Create(CoMutex, nullptr, "1"); c->Executor()->Create(CoMutex, nullptr, "2"); -} - -void TCoroTest::TestMutex() { - TContExecutor e(32000); - e.Execute(CoMutexTest); - UNIT_ASSERT_EQUAL(res, "1212"); +} + +void TCoroTest::TestMutex() { + TContExecutor e(32000); + e.Execute(CoMutexTest); + UNIT_ASSERT_EQUAL(res, "1212"); res.clear(); -} - -static TContMutex m1; -static TContCondVar c1; - -static void CoCondVar(TCont* c, void* /*run*/) { +} + +static TContMutex m1; +static TContCondVar c1; + +static void CoCondVar(TCont* c, void* /*run*/) { for (size_t i4 = 0; i4 < 3; ++i4) { - UNIT_ASSERT_EQUAL(m1.LockI(c), 0); - UNIT_ASSERT_EQUAL(c1.WaitI(c, &m1), 0); - res += c->Name(); - m1.UnLock(); - } -} - -static void CoCondVarTest(TCont* c, void* /*run*/) { + UNIT_ASSERT_EQUAL(m1.LockI(c), 0); + UNIT_ASSERT_EQUAL(c1.WaitI(c, &m1), 0); + res += c->Name(); + m1.UnLock(); + } +} + +static void CoCondVarTest(TCont* c, void* /*run*/) { c->Executor()->Create(CoCondVar, nullptr, "1"); - c->Yield(); + c->Yield(); c->Executor()->Create(CoCondVar, nullptr, "2"); - c->Yield(); + c->Yield(); c->Executor()->Create(CoCondVar, nullptr, "3"); - c->Yield(); + c->Yield(); c->Executor()->Create(CoCondVar, nullptr, "4"); - c->Yield(); + c->Yield(); c->Executor()->Create(CoCondVar, nullptr, "5"); - c->Yield(); + c->Yield(); c->Executor()->Create(CoCondVar, nullptr, "6"); - c->Yield(); - + c->Yield(); + for (size_t i5 = 0; i5 < 3; ++i5) { res += ToString((size_t)i5) + "^"; - c1.BroadCast(); - c->Yield(); - } -} - -void TCoroTest::TestCondVar() { - TContExecutor e(32000); - e.Execute(CoCondVarTest); - UNIT_ASSERT_EQUAL(res, "0^1234561^1234562^123456"); + c1.BroadCast(); + c->Yield(); + } +} + +void TCoroTest::TestCondVar() { + TContExecutor e(32000); + e.Execute(CoCondVarTest); + UNIT_ASSERT_EQUAL(res, "0^1234561^1234562^123456"); res.clear(); -} - +} + namespace NCoroTestJoin { struct TSleepCont { const TInstant Deadline; @@ -1004,4 +1004,4 @@ void TCoroTest::TestOverrideTime() { executor.Execute(); } -UNIT_TEST_SUITE_REGISTRATION(TCoroTest); +UNIT_TEST_SUITE_REGISTRATION(TCoroTest); diff --git a/library/cpp/coroutine/engine/events.h b/library/cpp/coroutine/engine/events.h index 07cc4d25e8..2e879999bf 100644 --- a/library/cpp/coroutine/engine/events.h +++ b/library/cpp/coroutine/engine/events.h @@ -1,148 +1,148 @@ #pragma once - + #include "impl.h" #include <util/datetime/base.h> -class TContEvent { -public: +class TContEvent { +public: TContEvent(TCont* current) noexcept - : Cont_(current) - , Status_(0) - { - } - + : Cont_(current) + , Status_(0) + { + } + ~TContEvent() { - } - + } + int WaitD(TInstant deadline) { - Status_ = 0; - const int ret = Cont_->SleepD(deadline); - - return Status_ ? Status_ : ret; - } - + Status_ = 0; + const int ret = Cont_->SleepD(deadline); + + return Status_ ? Status_ : ret; + } + int WaitT(TDuration timeout) { - return WaitD(timeout.ToDeadLine()); - } - + return WaitD(timeout.ToDeadLine()); + } + int WaitI() { - return WaitD(TInstant::Max()); - } - + return WaitD(TInstant::Max()); + } + void Wake() noexcept { - SetStatus(EWAKEDUP); - Cont_->ReSchedule(); - } - + SetStatus(EWAKEDUP); + Cont_->ReSchedule(); + } + TCont* Cont() noexcept { - return Cont_; - } - + return Cont_; + } + int Status() const noexcept { - return Status_; - } - + return Status_; + } + void SetStatus(int status) noexcept { - Status_ = status; - } - -private: - TCont* Cont_; - int Status_; -}; - -class TContWaitQueue { - class TWaiter: public TContEvent, public TIntrusiveListItem<TWaiter> { - public: + Status_ = status; + } + +private: + TCont* Cont_; + int Status_; +}; + +class TContWaitQueue { + class TWaiter: public TContEvent, public TIntrusiveListItem<TWaiter> { + public: TWaiter(TCont* current) noexcept - : TContEvent(current) - { - } - + : TContEvent(current) + { + } + ~TWaiter() { - } - }; - -public: + } + }; + +public: TContWaitQueue() noexcept { - } - + } + ~TContWaitQueue() { Y_ASSERT(Waiters_.Empty()); - } - + } + int WaitD(TCont* current, TInstant deadline) { - TWaiter waiter(current); - - Waiters_.PushBack(&waiter); - - return waiter.WaitD(deadline); - } - + TWaiter waiter(current); + + Waiters_.PushBack(&waiter); + + return waiter.WaitD(deadline); + } + int WaitT(TCont* current, TDuration timeout) { - return WaitD(current, timeout.ToDeadLine()); - } - + return WaitD(current, timeout.ToDeadLine()); + } + int WaitI(TCont* current) { - return WaitD(current, TInstant::Max()); - } - + return WaitD(current, TInstant::Max()); + } + void Signal() noexcept { - if (!Waiters_.Empty()) { - Waiters_.PopFront()->Wake(); - } - } + if (!Waiters_.Empty()) { + Waiters_.PopFront()->Wake(); + } + } void BroadCast() noexcept { - while (!Waiters_.Empty()) { - Waiters_.PopFront()->Wake(); - } - } - + while (!Waiters_.Empty()) { + Waiters_.PopFront()->Wake(); + } + } + void BroadCast(size_t number) noexcept { for (size_t i = 0; i < number && !Waiters_.Empty(); ++i) { Waiters_.PopFront()->Wake(); } } -private: - TIntrusiveList<TWaiter> Waiters_; -}; - +private: + TIntrusiveList<TWaiter> Waiters_; +}; + -class TContSimpleEvent { -public: +class TContSimpleEvent { +public: TContSimpleEvent(TContExecutor* e) - : E_(e) - { - } - + : E_(e) + { + } + TContExecutor* Executor() const noexcept { - return E_; - } - + return E_; + } + void Signal() noexcept { - Q_.Signal(); - } - + Q_.Signal(); + } + void BroadCast() noexcept { - Q_.BroadCast(); - } - + Q_.BroadCast(); + } + int WaitD(TInstant deadLine) noexcept { return Q_.WaitD(E_->Running(), deadLine); - } - + } + int WaitT(TDuration timeout) noexcept { - return WaitD(timeout.ToDeadLine()); - } - + return WaitD(timeout.ToDeadLine()); + } + int WaitI() noexcept { - return WaitD(TInstant::Max()); - } - -private: - TContWaitQueue Q_; - TContExecutor* E_; -}; + return WaitD(TInstant::Max()); + } + +private: + TContWaitQueue Q_; + TContExecutor* E_; +}; diff --git a/library/cpp/coroutine/engine/impl.cpp b/library/cpp/coroutine/engine/impl.cpp index 7ae6f74051..df08ddd3cc 100644 --- a/library/cpp/coroutine/engine/impl.cpp +++ b/library/cpp/coroutine/engine/impl.cpp @@ -1,15 +1,15 @@ #include "impl.h" - + #include "stack/stack_allocator.h" #include "stack/stack_guards.h" #include <util/generic/scope.h> #include <util/thread/singleton.h> #include <util/stream/format.h> -#include <util/stream/output.h> +#include <util/stream/output.h> #include <util/system/yassert.h> - - + + TCont::TJoinWait::TJoinWait(TCont& c) noexcept : Cont_(c) {} @@ -35,12 +35,12 @@ TCont::TCont(NCoro::NStack::IAllocator& allocator, void TCont::PrintMe(IOutputStream& out) const noexcept { - out << "cont(" + out << "cont(" << "name = " << Name_ << ", " << "addr = " << Hex((size_t)this) - << ")"; -} - + << ")"; +} + bool TCont::Join(TCont* c, TInstant deadLine) noexcept { TJoinWait ev(*this); c->Waiters_.PushBack(&ev); @@ -148,8 +148,8 @@ void TContExecutor::Execute(TContFunc func, void* arg) noexcept { func(cont, arg); }, "sys_main"); RunScheduler(); -} - +} + void TContExecutor::WaitForIO() { while (Ready_.Empty() && !WaitQueue_.Empty()) { const auto now = Now(); @@ -190,8 +190,8 @@ void TContExecutor::WaitForIO() { Ready_.Append(ReadyNext_); } -} - +} + void TContExecutor::Poll(TInstant deadline) { Poller_.Wait(PollerEvents_, deadline); LastPoll_ = Now(); @@ -217,8 +217,8 @@ void TContExecutor::Poll(TInstant deadline) { } } } -} - +} + void TContExecutor::Abort() noexcept { WaitQueue_.Abort(); auto visitor = [](TCont* c) { @@ -324,7 +324,7 @@ void TContExecutor::RunScheduler() noexcept { } TCont* cont = Ready_.PopFront(); - + if (ScheduleCallback_) { ScheduleCallback_->OnSchedule(*this, *cont); } @@ -347,9 +347,9 @@ void TContExecutor::RunScheduler() noexcept { } catch (...) { TBackTrace::FromCurrentException().PrintTo(Cerr); Y_FAIL("Uncaught exception in the scheduler: %s", CurrentExceptionMessage().c_str()); - } + } } - + void TContExecutor::Pause() { if (auto cont = Running()) { Paused_ = true; @@ -371,4 +371,4 @@ TInstant TContExecutor::Now() { template <> void Out<TCont>(IOutputStream& out, const TCont& c) { c.PrintMe(out); -} +} diff --git a/library/cpp/coroutine/engine/impl.h b/library/cpp/coroutine/engine/impl.h index 283a96ecf1..d904235ed3 100644 --- a/library/cpp/coroutine/engine/impl.h +++ b/library/cpp/coroutine/engine/impl.h @@ -1,30 +1,30 @@ #pragma once - + #include "callbacks.h" #include "cont_poller.h" #include "iostatus.h" -#include "poller.h" +#include "poller.h" #include "stack/stack_common.h" #include "trampoline.h" #include "custom_time.h" - + #include <library/cpp/containers/intrusive_rb_tree/rb_tree.h> -#include <util/system/error.h> -#include <util/generic/ptr.h> -#include <util/generic/intrlist.h> -#include <util/datetime/base.h> +#include <util/system/error.h> +#include <util/generic/ptr.h> +#include <util/generic/intrlist.h> +#include <util/datetime/base.h> #include <util/generic/maybe.h> #include <util/generic/function.h> - + #define EWAKEDUP 34567 -class TCont; -struct TContRep; -class TContExecutor; -class TContPollEvent; - +class TCont; +struct TContRep; +class TContExecutor; +class TContPollEvent; + namespace NCoro::NStack { class IAllocator; } @@ -32,18 +32,18 @@ namespace NCoro::NStack { class TCont : private TIntrusiveListItem<TCont> { struct TJoinWait: public TIntrusiveListItem<TJoinWait> { TJoinWait(TCont& c) noexcept; - + void Wake() noexcept; - + public: TCont& Cont_; }; - + friend class TContExecutor; friend class TIntrusiveListItem<TCont>; friend class NCoro::TEventWaitQueue; friend class NCoro::TTrampoline; - + private: TCont( NCoro::NStack::IAllocator& allocator, @@ -52,63 +52,63 @@ private: NCoro::TTrampoline::TFunc func, const char* name ) noexcept; - + public: TContExecutor* Executor() noexcept { return &Executor_; - } - + } + const TContExecutor* Executor() const noexcept { return &Executor_; - } - + } + const char* Name() const noexcept { - return Name_; - } - + return Name_; + } + void PrintMe(IOutputStream& out) const noexcept; - + void Yield() noexcept; - + void ReScheduleAndSwitch() noexcept; - - /// @return ETIMEDOUT on success + + /// @return ETIMEDOUT on success int SleepD(TInstant deadline) noexcept; - + int SleepT(TDuration timeout) noexcept { - return SleepD(timeout.ToDeadLine()); - } - + return SleepD(timeout.ToDeadLine()); + } + int SleepI() noexcept { - return SleepD(TInstant::Max()); - } - + return SleepD(TInstant::Max()); + } + bool IAmRunning() const noexcept; void Cancel() noexcept; bool Cancelled() const noexcept { - return Cancelled_; - } - + return Cancelled_; + } + bool Scheduled() const noexcept { return Scheduled_; } bool Join(TCont* c, TInstant deadLine = TInstant::Max()) noexcept; - + void ReSchedule() noexcept; - + void Switch() noexcept; void SwitchTo(TExceptionSafeContext* ctx) { Trampoline_.SwitchTo(ctx); } -private: +private: void Terminate(); -private: +private: TContExecutor& Executor_; // TODO(velavokr): allow name storage owning (for generated names backed by TString) @@ -119,17 +119,17 @@ private: TIntrusiveList<TJoinWait> Waiters_; bool Cancelled_ = false; bool Scheduled_ = false; -}; - +}; + TCont* RunningCont(); - - -template <class Functor> -static void ContHelperFunc(TCont* cont, void* arg) { - (*((Functor*)(arg)))(cont); -} - -template <typename T, void (T::*M)(TCont*)> + + +template <class Functor> +static void ContHelperFunc(TCont* cont, void* arg) { + (*((Functor*)(arg)))(cont); +} + +template <typename T, void (T::*M)(TCont*)> static void ContHelperMemberFunc(TCont* c, void* arg) { ((reinterpret_cast<T*>(arg))->*M)(c); } @@ -145,11 +145,11 @@ public: /// Central coroutine class. /// Note, coroutines are single-threaded, and all methods must be called from the single thread -class TContExecutor { - friend class TCont; +class TContExecutor { + friend class TCont; using TContList = TIntrusiveList<TCont>; - -public: + +public: TContExecutor( uint32_t defaultStackSize, THolder<IPollerFace> poller = IPollerFace::Default(), @@ -159,41 +159,41 @@ public: TMaybe<NCoro::NStack::TPoolAllocatorSettings> poolSettings = Nothing(), NCoro::ITime* time = nullptr ); - + ~TContExecutor(); - + // if we already have a coroutine to run void Execute() noexcept; - + void Execute(TContFunc func, void* arg = nullptr) noexcept; - - template <class Functor> + + template <class Functor> void Execute(Functor& f) noexcept { - Execute((TContFunc)ContHelperFunc<Functor>, (void*)&f); - } - - template <typename T, void (T::*M)(TCont*)> + Execute((TContFunc)ContHelperFunc<Functor>, (void*)&f); + } + + template <typename T, void (T::*M)(TCont*)> void Execute(T* obj) noexcept { - Execute(ContHelperMemberFunc<T, M>, obj); - } + Execute(ContHelperMemberFunc<T, M>, obj); + } - template <class Functor> + template <class Functor> TCont* Create( Functor& f, const char* name, TMaybe<ui32> customStackSize = Nothing() ) noexcept { return Create((TContFunc)ContHelperFunc<Functor>, (void*)&f, name, customStackSize); - } - - template <typename T, void (T::*M)(TCont*)> + } + + template <typename T, void (T::*M)(TCont*)> TCont* Create( T* obj, const char* name, TMaybe<ui32> customStackSize = Nothing() ) noexcept { return Create(ContHelperMemberFunc<T, M>, obj, name, customStackSize); - } + } TCont* Create( TContFunc func, @@ -201,7 +201,7 @@ public: const char* name, TMaybe<ui32> customStackSize = Nothing() ) noexcept; - + TCont* CreateOwned( NCoro::TTrampoline::TFunc func, const char* name, @@ -209,17 +209,17 @@ public: ) noexcept; NCoro::TContPoller* Poller() noexcept { - return &Poller_; - } - + return &Poller_; + } + TCont* Running() noexcept { - return Current_; - } - + return Current_; + } + const TCont* Running() const noexcept { - return Current_; - } - + return Current_; + } + size_t TotalReadyConts() const noexcept { return Ready_.Size() + TotalScheduledConts(); } @@ -240,54 +240,54 @@ public: // TODO(velavokr): rename, it is just CancelAll actually void Abort() noexcept; - + void SetFailOnError(bool fail) noexcept { - FailOnError_ = fail; - } - + FailOnError_ = fail; + } + bool FailOnError() const noexcept { - return FailOnError_; - } - + return FailOnError_; + } + void RegisterInWaitQueue(NCoro::TContPollEvent* event) { WaitQueue_.Register(event); } void ScheduleIoWait(TFdEvent* event) { RegisterInWaitQueue(event); - Poller_.Schedule(event); - } - + Poller_.Schedule(event); + } + void ScheduleIoWait(TTimerEvent* event) noexcept { RegisterInWaitQueue(event); - } - + } + void ScheduleUserEvent(IUserEvent* event) { UserEvents_.PushBack(event); } void Pause(); TInstant Now(); -private: +private: void Release(TCont* cont) noexcept; - + void Exit(TCont* cont) noexcept; - + void RunScheduler() noexcept; - + void ScheduleToDelete(TCont* cont) noexcept; - + void ScheduleExecution(TCont* cont) noexcept; - + void ScheduleExecutionNow(TCont* cont) noexcept; void DeleteScheduled() noexcept; - - void WaitForIO(); - + + void WaitForIO(); + void Poll(TInstant deadline); -private: +private: NCoro::IScheduleCallback* const ScheduleCallback_ = nullptr; NCoro::IEnterPollerCallback* const EnterPollerCallback_ = nullptr; const uint32_t DefaultStackSize_; @@ -295,8 +295,8 @@ private: TExceptionSafeContext SchedContext_; - TContList ToDelete_; - TContList Ready_; + TContList ToDelete_; + TContList Ready_; TContList ReadyNext_; NCoro::TEventWaitQueue WaitQueue_; NCoro::TContPoller Poller_; @@ -310,4 +310,4 @@ private: bool FailOnError_ = false; bool Paused_ = false; NCoro::ITime* Time_ = nullptr; -}; +}; diff --git a/library/cpp/coroutine/engine/iostatus.cpp b/library/cpp/coroutine/engine/iostatus.cpp index 8298229b39..a92142edeb 100644 --- a/library/cpp/coroutine/engine/iostatus.cpp +++ b/library/cpp/coroutine/engine/iostatus.cpp @@ -1 +1 @@ -#include "iostatus.h" +#include "iostatus.h" diff --git a/library/cpp/coroutine/engine/iostatus.h b/library/cpp/coroutine/engine/iostatus.h index bf6036805d..8478ce05e8 100644 --- a/library/cpp/coroutine/engine/iostatus.h +++ b/library/cpp/coroutine/engine/iostatus.h @@ -1,91 +1,91 @@ #pragma once - + #include <util/generic/yexception.h> -class TIOStatus { -public: +class TIOStatus { +public: TIOStatus(int status) noexcept - : Status_(status) - { - } - + : Status_(status) + { + } + static TIOStatus Error(int status) noexcept { - return TIOStatus(status); - } - + return TIOStatus(status); + } + static TIOStatus Error() noexcept { - return TIOStatus(LastSystemError()); - } - + return TIOStatus(LastSystemError()); + } + static TIOStatus Success() noexcept { - return TIOStatus(0); - } - + return TIOStatus(0); + } + void Check() const { - if (Status_) { - ythrow TSystemError(Status_) << "io error"; - } - } - + if (Status_) { + ythrow TSystemError(Status_) << "io error"; + } + } + bool Failed() const noexcept { - return (bool)Status_; - } - + return (bool)Status_; + } + bool Succeed() const noexcept { - return !Failed(); - } - + return !Failed(); + } + int Status() const noexcept { - return Status_; - } - -private: - int Status_; -}; - - -class TContIOStatus { -public: + return Status_; + } + +private: + int Status_; +}; + + +class TContIOStatus { +public: TContIOStatus(size_t processed, TIOStatus status) noexcept - : Processed_(processed) - , Status_(status) - { - } - + : Processed_(processed) + , Status_(status) + { + } + static TContIOStatus Error(TIOStatus status) noexcept { - return TContIOStatus(0, status); - } - + return TContIOStatus(0, status); + } + static TContIOStatus Error() noexcept { - return TContIOStatus(0, TIOStatus::Error()); - } - + return TContIOStatus(0, TIOStatus::Error()); + } + static TContIOStatus Success(size_t processed) noexcept { - return TContIOStatus(processed, TIOStatus::Success()); - } - + return TContIOStatus(processed, TIOStatus::Success()); + } + static TContIOStatus Eof() noexcept { - return Success(0); - } - + return Success(0); + } + ~TContIOStatus() { - } - + } + size_t Processed() const noexcept { - return Processed_; - } - + return Processed_; + } + int Status() const noexcept { - return Status_.Status(); - } - + return Status_.Status(); + } + size_t Checked() const { - Status_.Check(); - - return Processed_; - } - -private: - size_t Processed_; - TIOStatus Status_; -}; + Status_.Check(); + + return Processed_; + } + +private: + size_t Processed_; + TIOStatus Status_; +}; diff --git a/library/cpp/coroutine/engine/mutex.h b/library/cpp/coroutine/engine/mutex.h index 93e9119503..27392f7d85 100644 --- a/library/cpp/coroutine/engine/mutex.h +++ b/library/cpp/coroutine/engine/mutex.h @@ -1,49 +1,49 @@ #pragma once - + #include "impl.h" #include "events.h" -class TContMutex { -public: +class TContMutex { +public: TContMutex() noexcept - : Token_(true) - { - } - + : Token_(true) + { + } + ~TContMutex() { Y_ASSERT(Token_); - } - + } + int LockD(TCont* current, TInstant deadline) { - while (!Token_) { - const int ret = WaitQueue_.WaitD(current, deadline); - - if (ret != EWAKEDUP) { - return ret; - } - } - - Token_ = false; - - return 0; - } - + while (!Token_) { + const int ret = WaitQueue_.WaitD(current, deadline); + + if (ret != EWAKEDUP) { + return ret; + } + } + + Token_ = false; + + return 0; + } + int LockT(TCont* current, TDuration timeout) { - return LockD(current, timeout.ToDeadLine()); - } - + return LockD(current, timeout.ToDeadLine()); + } + int LockI(TCont* current) { - return LockD(current, TInstant::Max()); - } - + return LockD(current, TInstant::Max()); + } + void UnLock() noexcept { Y_ASSERT(!Token_); - - Token_ = true; - WaitQueue_.Signal(); - } - -private: - TContWaitQueue WaitQueue_; - bool Token_; -}; + + Token_ = true; + WaitQueue_.Signal(); + } + +private: + TContWaitQueue WaitQueue_; + bool Token_; +}; diff --git a/library/cpp/coroutine/engine/network.cpp b/library/cpp/coroutine/engine/network.cpp index 85b647d210..a5c0d9282c 100644 --- a/library/cpp/coroutine/engine/network.cpp +++ b/library/cpp/coroutine/engine/network.cpp @@ -1,41 +1,41 @@ #include "impl.h" #include "network.h" - + #include <util/generic/scope.h> #include <util/generic/xrange.h> - + #include <sys/uio.h> - + #if defined(_bionic_) # define IOV_MAX 1024 #endif - - + + namespace NCoro { namespace { bool IsBlocked(int lasterr) noexcept { return lasterr == EAGAIN || lasterr == EWOULDBLOCK; } - + ssize_t DoReadVector(SOCKET fd, TContIOVector* vec) noexcept { return readv(fd, (const iovec*) vec->Parts(), Min(IOV_MAX, (int) vec->Count())); } - + ssize_t DoWriteVector(SOCKET fd, TContIOVector* vec) noexcept { return writev(fd, (const iovec*) vec->Parts(), Min(IOV_MAX, (int) vec->Count())); } } - - + + int SelectD(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd, TInstant deadline) noexcept { if (cont->Cancelled()) { return ECANCELED; } - + if (nfds == 0) { return 0; } - + TTempArray<TFdEvent> events(nfds); for (auto i : xrange(nfds)) { @@ -47,7 +47,7 @@ namespace NCoro { (events.Data() + i)->~TFdEvent(); } }; - + for (auto i : xrange(nfds)) { cont->Executor()->ScheduleIoWait(events.Data() + i); } @@ -79,36 +79,36 @@ namespace NCoro { if (ret) { if (outfd) { *outfd = ret->Fd(); - } + } return ret->Status(); - } - + } + return EINPROGRESS; - } - + } + int SelectT(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd, TDuration timeout) noexcept { return SelectD(cont, fds, what, nfds, outfd, timeout.ToDeadLine()); - } - + } + int SelectI(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd) { return SelectD(cont, fds, what, nfds, outfd, TInstant::Max()); } - + int PollD(TCont* cont, SOCKET fd, int what, TInstant deadline) noexcept { TFdEvent event(cont, fd, (ui16)what, deadline); return ExecuteEvent(&event); - } - + } + int PollT(TCont* cont, SOCKET fd, int what, TDuration timeout) noexcept { return PollD(cont, fd, what, timeout.ToDeadLine()); } int PollI(TCont* cont, SOCKET fd, int what) noexcept { return PollD(cont, fd, what, TInstant::Max()); - } - - + } + + TContIOStatus ReadVectorD(TCont* cont, SOCKET fd, TContIOVector* vec, TInstant deadline) noexcept { while (true) { ssize_t res = DoReadVector(fd, vec); @@ -117,103 +117,103 @@ namespace NCoro { return TContIOStatus::Success((size_t) res); } - { - const int err = LastSystemError(); - - if (!IsBlocked(err)) { + { + const int err = LastSystemError(); + + if (!IsBlocked(err)) { return TContIOStatus::Error(err); - } - } - + } + } + if ((res = PollD(cont, fd, CONT_POLL_READ, deadline)) != 0) { return TContIOStatus::Error((int) res); - } - } - } - + } + } + } + TContIOStatus ReadVectorT(TCont* cont, SOCKET fd, TContIOVector* vec, TDuration timeOut) noexcept { return ReadVectorD(cont, fd, vec, timeOut.ToDeadLine()); } - + TContIOStatus ReadVectorI(TCont* cont, SOCKET fd, TContIOVector* vec) noexcept { return ReadVectorD(cont, fd, vec, TInstant::Max()); } - - + + TContIOStatus ReadD(TCont* cont, SOCKET fd, void* buf, size_t len, TInstant deadline) noexcept { IOutputStream::TPart part(buf, len); TContIOVector vec(&part, 1); return ReadVectorD(cont, fd, &vec, deadline); } - + TContIOStatus ReadT(TCont* cont, SOCKET fd, void* buf, size_t len, TDuration timeout) noexcept { return ReadD(cont, fd, buf, len, timeout.ToDeadLine()); } - + TContIOStatus ReadI(TCont* cont, SOCKET fd, void* buf, size_t len) noexcept { return ReadD(cont, fd, buf, len, TInstant::Max()); - } - - + } + + TContIOStatus WriteVectorD(TCont* cont, SOCKET fd, TContIOVector* vec, TInstant deadline) noexcept { size_t written = 0; - + while (!vec->Complete()) { ssize_t res = DoWriteVector(fd, vec); - + if (res >= 0) { written += res; - + vec->Proceed((size_t) res); } else { { const int err = LastSystemError(); - + if (!IsBlocked(err)) { return TContIOStatus(written, err); } } - + if ((res = PollD(cont, fd, CONT_POLL_WRITE, deadline)) != 0) { return TContIOStatus(written, (int) res); } } - } - + } + return TContIOStatus::Success(written); - } - + } + TContIOStatus WriteVectorT(TCont* cont, SOCKET fd, TContIOVector* vec, TDuration timeOut) noexcept { return WriteVectorD(cont, fd, vec, timeOut.ToDeadLine()); } - + TContIOStatus WriteVectorI(TCont* cont, SOCKET fd, TContIOVector* vec) noexcept { return WriteVectorD(cont, fd, vec, TInstant::Max()); } - - + + TContIOStatus WriteD(TCont* cont, SOCKET fd, const void* buf, size_t len, TInstant deadline) noexcept { IOutputStream::TPart part(buf, len); TContIOVector vec(&part, 1); return WriteVectorD(cont, fd, &vec, deadline); } - + TContIOStatus WriteT(TCont* cont, SOCKET fd, const void* buf, size_t len, TDuration timeout) noexcept { return WriteD(cont, fd, buf, len, timeout.ToDeadLine()); } - + TContIOStatus WriteI(TCont* cont, SOCKET fd, const void* buf, size_t len) noexcept { return WriteD(cont, fd, buf, len, TInstant::Max()); - } - - + } + + int ConnectD(TCont* cont, TSocketHolder& s, const struct addrinfo& ai, TInstant deadline) noexcept { TSocketHolder res(Socket(ai)); - + if (res.Closed()) { return LastSystemError(); } - + const int ret = ConnectD(cont, res, ai.ai_addr, (socklen_t) ai.ai_addrlen, deadline); if (!ret) { @@ -221,49 +221,49 @@ namespace NCoro { } return ret; - } - + } + int ConnectD(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr, TInstant deadline) noexcept { int ret = EHOSTUNREACH; - + for (auto it = addr.Begin(); it != addr.End(); ++it) { ret = ConnectD(cont, s, *it, deadline); - + if (ret == 0 || ret == ETIMEDOUT) { return ret; } } - + return ret; - } - + } + int ConnectT(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr, TDuration timeout) noexcept { return ConnectD(cont, s, addr, timeout.ToDeadLine()); } - + int ConnectI(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr) noexcept { return ConnectD(cont, s, addr, TInstant::Max()); - } - + } + int ConnectD(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen, TInstant deadline) noexcept { if (connect(s, name, namelen)) { const int err = LastSystemError(); - + if (!IsBlocked(err) && err != EINPROGRESS) { return err; } - + int ret = PollD(cont, s, CONT_POLL_WRITE, deadline); - + if (ret) { return ret; } - + // check if we really connected // FIXME: Unportable ?? int serr = 0; socklen_t slen = sizeof(serr); - + ret = getsockopt(s, SOL_SOCKET, SO_ERROR, (char*) &serr, &slen); if (ret) { @@ -286,27 +286,27 @@ namespace NCoro { return ConnectD(cont, s, name, namelen, TInstant::Max()); } - + int AcceptD(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen, TInstant deadline) noexcept { SOCKET ret; - + while ((ret = Accept4(s, addr, addrlen)) == INVALID_SOCKET) { int err = LastSystemError(); - + if (!IsBlocked(err)) { return -err; } err = PollD(cont, s, CONT_POLL_READ, deadline); - + if (err) { return -err; } } - + return (int) ret; - } - + } + int AcceptT(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen, TDuration timeout) noexcept { return AcceptD(cont, s, addr, addrlen, timeout.ToDeadLine()); } diff --git a/library/cpp/coroutine/engine/network.h b/library/cpp/coroutine/engine/network.h index f2c9afe4f8..db979c4c59 100644 --- a/library/cpp/coroutine/engine/network.h +++ b/library/cpp/coroutine/engine/network.h @@ -1,55 +1,55 @@ #pragma once - + #include "iostatus.h" - + #include <util/datetime/base.h> #include <util/network/init.h> #include <util/network/iovec.h> #include <util/network/nonblock.h> -#include <util/network/socket.h> - -class TCont; - +#include <util/network/socket.h> + +class TCont; + namespace NCoro { int SelectD(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd, TInstant deadline) noexcept; int SelectT(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd, TDuration timeout) noexcept; int SelectI(TCont* cont, SOCKET fds[], int what[], size_t nfds, SOCKET* outfd); - + int PollD(TCont* cont, SOCKET fd, int what, TInstant deadline) noexcept; int PollT(TCont* cont, SOCKET fd, int what, TDuration timeout) noexcept; int PollI(TCont* cont, SOCKET fd, int what) noexcept; - + TContIOStatus ReadVectorD(TCont* cont, SOCKET fd, TContIOVector* vec, TInstant deadline) noexcept; TContIOStatus ReadVectorT(TCont* cont, SOCKET fd, TContIOVector* vec, TDuration timeOut) noexcept; TContIOStatus ReadVectorI(TCont* cont, SOCKET fd, TContIOVector* vec) noexcept; - + TContIOStatus ReadD(TCont* cont, SOCKET fd, void* buf, size_t len, TInstant deadline) noexcept; TContIOStatus ReadT(TCont* cont, SOCKET fd, void* buf, size_t len, TDuration timeout) noexcept; TContIOStatus ReadI(TCont* cont, SOCKET fd, void* buf, size_t len) noexcept; - + TContIOStatus WriteVectorD(TCont* cont, SOCKET fd, TContIOVector* vec, TInstant deadline) noexcept; TContIOStatus WriteVectorT(TCont* cont, SOCKET fd, TContIOVector* vec, TDuration timeOut) noexcept; TContIOStatus WriteVectorI(TCont* cont, SOCKET fd, TContIOVector* vec) noexcept; - + TContIOStatus WriteD(TCont* cont, SOCKET fd, const void* buf, size_t len, TInstant deadline) noexcept; TContIOStatus WriteT(TCont* cont, SOCKET fd, const void* buf, size_t len, TDuration timeout) noexcept; TContIOStatus WriteI(TCont* cont, SOCKET fd, const void* buf, size_t len) noexcept; - + int ConnectD(TCont* cont, TSocketHolder& s, const struct addrinfo& ai, TInstant deadline) noexcept; - + int ConnectD(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr, TInstant deadline) noexcept; int ConnectT(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr, TDuration timeout) noexcept; int ConnectI(TCont* cont, TSocketHolder& s, const TNetworkAddress& addr) noexcept; - + int ConnectD(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen, TInstant deadline) noexcept; int ConnectT(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen, TDuration timeout) noexcept; int ConnectI(TCont* cont, SOCKET s, const struct sockaddr* name, socklen_t namelen) noexcept; - + int AcceptD(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen, TInstant deadline) noexcept; int AcceptT(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen, TDuration timeout) noexcept; int AcceptI(TCont* cont, SOCKET s, struct sockaddr* addr, socklen_t* addrlen) noexcept; - + SOCKET Socket(int domain, int type, int protocol) noexcept; SOCKET Socket(const struct addrinfo& ai) noexcept; -} +} diff --git a/library/cpp/coroutine/engine/poller.cpp b/library/cpp/coroutine/engine/poller.cpp index 61164fa56b..6fd1f673ae 100644 --- a/library/cpp/coroutine/engine/poller.cpp +++ b/library/cpp/coroutine/engine/poller.cpp @@ -1,118 +1,118 @@ -#include "poller.h" -#include "sockmap.h" - -#include <util/memory/smallobj.h> -#include <util/generic/intrlist.h> -#include <util/generic/singleton.h> +#include "poller.h" +#include "sockmap.h" + +#include <util/memory/smallobj.h> +#include <util/generic/intrlist.h> +#include <util/generic/singleton.h> #include <util/system/env.h> #include <util/string/cast.h> - -namespace { + +namespace { using TChange = IPollerFace::TChange; using TEvent = IPollerFace::TEvent; using TEvents = IPollerFace::TEvents; - - template <class T> - class TUnsafeBuf { - public: + + template <class T> + class TUnsafeBuf { + public: TUnsafeBuf() noexcept - : L_(0) - { - } - + : L_(0) + { + } + T* operator~() const noexcept { - return B_.Get(); - } - + return B_.Get(); + } + size_t operator+() const noexcept { - return L_; - } - + return L_; + } + void Reserve(size_t len) { - len = FastClp2(len); - - if (len > L_) { - B_.Reset(new T[len]); - L_ = len; - } - } - - private: - TArrayHolder<T> B_; - size_t L_; - }; - - - template <class T> - class TVirtualize: public IPollerFace { - public: + len = FastClp2(len); + + if (len > L_) { + B_.Reset(new T[len]); + L_ = len; + } + } + + private: + TArrayHolder<T> B_; + size_t L_; + }; + + + template <class T> + class TVirtualize: public IPollerFace { + public: TVirtualize(EContPoller pollerEngine) : PollerEngine_(pollerEngine) { } void Set(const TChange& c) override { - P_.Set(c); - } - + P_.Set(c); + } + void Wait(TEvents& events, TInstant deadLine) override { - P_.Wait(events, deadLine); - } - + P_.Wait(events, deadLine); + } + EContPoller PollEngine() const override { return PollerEngine_; } - private: - T P_; + private: + T P_; const EContPoller PollerEngine_; - }; - + }; + - template <class T> - class TPoller { + template <class T> + class TPoller { using TInternalEvent = typename T::TEvent; - - public: + + public: TPoller() { - E_.Reserve(1); - } - + E_.Reserve(1); + } + void Set(const TChange& c) { - P_.Set(c.Data, c.Fd, c.Flags); - } - + P_.Set(c.Data, c.Fd, c.Flags); + } + void Reserve(size_t size) { E_.Reserve(size); } void Wait(TEvents& events, TInstant deadLine) { - const size_t ret = P_.WaitD(~E_, +E_, deadLine); - + const size_t ret = P_.WaitD(~E_, +E_, deadLine); + events.reserve(ret); - for (size_t i = 0; i < ret; ++i) { - const TInternalEvent* ie = ~E_ + i; - - const TEvent e = { - T::ExtractEvent(ie), - T::ExtractStatus(ie), - (ui16)T::ExtractFilter(ie), - }; - - events.push_back(e); - } - - E_.Reserve(ret + 1); - } - - private: - T P_; - TUnsafeBuf<TInternalEvent> E_; - }; - - - template <class T> - class TIndexedArray { + for (size_t i = 0; i < ret; ++i) { + const TInternalEvent* ie = ~E_ + i; + + const TEvent e = { + T::ExtractEvent(ie), + T::ExtractStatus(ie), + (ui16)T::ExtractFilter(ie), + }; + + events.push_back(e); + } + + E_.Reserve(ret + 1); + } + + private: + T P_; + TUnsafeBuf<TInternalEvent> E_; + }; + + + template <class T> + class TIndexedArray { struct TVal: public T, public TIntrusiveListItem<TVal>, @@ -125,244 +125,244 @@ namespace { // zero-initialization takes place in TVal() expression and the // data is overwritten. TVal() { - } - }; - + } + }; + typedef TIntrusiveList<TVal> TListType; - - public: + + public: typedef typename TListType::TIterator TIterator; typedef typename TListType::TConstIterator TConstIterator; - + TIndexedArray() - : P_(TMemoryPool::TExpGrow::Instance(), TDefaultAllocator::Instance()) - { - } - + : P_(TMemoryPool::TExpGrow::Instance(), TDefaultAllocator::Instance()) + { + } + TIterator Begin() noexcept { - return I_.Begin(); - } - + return I_.Begin(); + } + TIterator End() noexcept { - return I_.End(); - } - + return I_.End(); + } + TConstIterator Begin() const noexcept { - return I_.Begin(); - } - + return I_.Begin(); + } + TConstIterator End() const noexcept { - return I_.End(); - } - + return I_.End(); + } + T& operator[](size_t i) { - return *Get(i); - } - + return *Get(i); + } + T* Get(size_t i) { - TValRef& v = V_.Get(i); - + TValRef& v = V_.Get(i); + if (Y_UNLIKELY(!v)) { - v.Reset(new (&P_) TVal()); - I_.PushFront(v.Get()); - } - - Y_PREFETCH_WRITE(v.Get(), 1); - - return v.Get(); - } - + v.Reset(new (&P_) TVal()); + I_.PushFront(v.Get()); + } + + Y_PREFETCH_WRITE(v.Get(), 1); + + return v.Get(); + } + void Erase(size_t i) noexcept { - V_.Get(i).Destroy(); - } - + V_.Get(i).Destroy(); + } + size_t Size() const noexcept { - return I_.Size(); - } - - private: + return I_.Size(); + } + + private: using TValRef = THolder<TVal>; - typename TVal::TPool P_; - TSocketMap<TValRef> V_; + typename TVal::TPool P_; + TSocketMap<TValRef> V_; TListType I_; - }; - + }; + inline short PollFlags(ui16 flags) noexcept { - short ret = 0; - - if (flags & CONT_POLL_READ) { - ret |= POLLIN; - } - - if (flags & CONT_POLL_WRITE) { - ret |= POLLOUT; - } - + short ret = 0; + + if (flags & CONT_POLL_READ) { + ret |= POLLIN; + } + + if (flags & CONT_POLL_WRITE) { + ret |= POLLOUT; + } + #if defined(_linux_) if (flags & CONT_POLL_RDHUP) { ret |= POLLRDHUP; } #endif - return ret; - } - + return ret; + } + - class TPollPoller { - public: + class TPollPoller { + public: size_t Size() const noexcept { - return S_.Size(); - } - - template <class T> + return S_.Size(); + } + + template <class T> void Build(T& t) const { - for (TFds::TConstIterator it = S_.Begin(); it != S_.End(); ++it) { - t.Set(*it); - } + for (TFds::TConstIterator it = S_.Begin(); it != S_.End(); ++it) { + t.Set(*it); + } t.Reserve(Size()); - } - + } + void Set(const TChange& c) { - if (c.Flags) { - S_[c.Fd] = c; - } else { - S_.Erase(c.Fd); - } - } - + if (c.Flags) { + S_[c.Fd] = c; + } else { + S_.Erase(c.Fd); + } + } + void Wait(TEvents& events, TInstant deadLine) { - T_.clear(); - T_.reserve(Size()); - - for (TFds::TConstIterator it = S_.Begin(); it != S_.End(); ++it) { - const pollfd pfd = { - it->Fd, - PollFlags(it->Flags), - 0, - }; - - T_.push_back(pfd); - } - + T_.clear(); + T_.reserve(Size()); + + for (TFds::TConstIterator it = S_.Begin(); it != S_.End(); ++it) { + const pollfd pfd = { + it->Fd, + PollFlags(it->Flags), + 0, + }; + + T_.push_back(pfd); + } + const ssize_t ret = PollD(T_.data(), (nfds_t) T_.size(), deadLine); - + if (ret <= 0) { - return; - } - + return; + } + events.reserve(T_.size()); for (size_t i = 0; i < T_.size(); ++i) { - const pollfd& pfd = T_[i]; - const short ev = pfd.revents; - - if (!ev) { - continue; - } - - int status = 0; - ui16 filter = 0; - + const pollfd& pfd = T_[i]; + const short ev = pfd.revents; + + if (!ev) { + continue; + } + + int status = 0; + ui16 filter = 0; + // We are perfectly fine with an EOF while reading a pipe or a unix socket if ((ev & POLLIN) || (ev & POLLHUP) && (pfd.events & POLLIN)) { - filter |= CONT_POLL_READ; - } - - if (ev & POLLOUT) { - filter |= CONT_POLL_WRITE; - } - + filter |= CONT_POLL_READ; + } + + if (ev & POLLOUT) { + filter |= CONT_POLL_WRITE; + } + #if defined(_linux_) if (ev & POLLRDHUP) { filter |= CONT_POLL_RDHUP; } #endif - if (ev & POLLERR) { - status = EIO; + if (ev & POLLERR) { + status = EIO; } else if (ev & POLLHUP && pfd.events & POLLOUT) { // Only write operations may cause EPIPE - status = EPIPE; - } else if (ev & POLLNVAL) { - status = EINVAL; - } - - if (status) { + status = EPIPE; + } else if (ev & POLLNVAL) { + status = EINVAL; + } + + if (status) { filter = CONT_POLL_READ | CONT_POLL_WRITE | CONT_POLL_RDHUP; - } - - const TEvent res = { - S_[pfd.fd].Data, - status, - filter, - }; - - events.push_back(res); - } - } - - private: - typedef TIndexedArray<TChange> TFds; - TFds S_; + } + + const TEvent res = { + S_[pfd.fd].Data, + status, + filter, + }; + + events.push_back(res); + } + } + + private: + typedef TIndexedArray<TChange> TFds; + TFds S_; typedef TVector<pollfd> TPollVec; - TPollVec T_; - }; - - - class TCombinedPoller { - typedef TPoller<TPollerImpl<TWithoutLocking>> TDefaultPoller; - - public: + TPollVec T_; + }; + + + class TCombinedPoller { + typedef TPoller<TPollerImpl<TWithoutLocking>> TDefaultPoller; + + public: TCombinedPoller() { - P_.Reset(new TPollPoller()); - } - + P_.Reset(new TPollPoller()); + } + void Set(const TChange& c) { - if (!P_) { - D_->Set(c); - } else { - P_->Set(c); - } - } - + if (!P_) { + D_->Set(c); + } else { + P_->Set(c); + } + } + void Wait(TEvents& events, TInstant deadLine) { - if (!P_) { - D_->Wait(events, deadLine); - } else { - if (P_->Size() > 200) { - D_.Reset(new TDefaultPoller()); - P_->Build(*D_); - P_.Destroy(); - D_->Wait(events, deadLine); - } else { - P_->Wait(events, deadLine); - } - } - } - - private: + if (!P_) { + D_->Wait(events, deadLine); + } else { + if (P_->Size() > 200) { + D_.Reset(new TDefaultPoller()); + P_->Build(*D_); + P_.Destroy(); + D_->Wait(events, deadLine); + } else { + P_->Wait(events, deadLine); + } + } + } + + private: THolder<TPollPoller> P_; THolder<TDefaultPoller> D_; - }; - - struct TUserPoller: public TString { + }; + + struct TUserPoller: public TString { TUserPoller() - : TString(GetEnv("USER_POLLER")) - { - } - }; -} - + : TString(GetEnv("USER_POLLER")) + { + } + }; +} + THolder<IPollerFace> IPollerFace::Default() { return Construct(*SingletonWithPriority<TUserPoller, 0>()); -} - +} + THolder<IPollerFace> IPollerFace::Construct(TStringBuf name) { return Construct(name ? FromString<EContPoller>(name) : EContPoller::Default); } - + THolder<IPollerFace> IPollerFace::Construct(EContPoller poller) { switch (poller) { case EContPoller::Default: @@ -373,18 +373,18 @@ THolder<IPollerFace> IPollerFace::Construct(EContPoller poller) { case EContPoller::Poll: return MakeHolder<TVirtualize<TPollPoller>>(poller); case EContPoller::Epoll: -#if defined(HAVE_EPOLL_POLLER) +#if defined(HAVE_EPOLL_POLLER) return MakeHolder<TVirtualize<TPoller<TGenericPoller<TEpollPoller<TWithoutLocking>>>>>(poller); #else return nullptr; -#endif +#endif case EContPoller::Kqueue: #if defined(HAVE_KQUEUE_POLLER) return MakeHolder<TVirtualize<TPoller<TGenericPoller<TKqueuePoller<TWithoutLocking>>>>>(poller); #else return nullptr; -#endif +#endif default: Y_FAIL("bad poller type"); - } -} + } +} diff --git a/library/cpp/coroutine/engine/poller.h b/library/cpp/coroutine/engine/poller.h index 8ea012c0fc..054de6616b 100644 --- a/library/cpp/coroutine/engine/poller.h +++ b/library/cpp/coroutine/engine/poller.h @@ -1,11 +1,11 @@ #pragma once - -#include <util/generic/ptr.h> -#include <util/generic/vector.h> -#include <util/network/socket.h> -#include <util/network/pollerimpl.h> -#include <util/datetime/base.h> - + +#include <util/generic/ptr.h> +#include <util/generic/vector.h> +#include <util/network/socket.h> +#include <util/network/pollerimpl.h> +#include <util/datetime/base.h> + enum class EContPoller { Default /* "default" */, Combined /* "combined" */, @@ -15,36 +15,36 @@ enum class EContPoller { Kqueue /* "kqueue" */ }; -class IPollerFace { -public: - struct TChange { +class IPollerFace { +public: + struct TChange { SOCKET Fd; - void* Data; - ui16 Flags; - }; - - struct TEvent { - void* Data; - int Status; - ui16 Filter; - }; - + void* Data; + ui16 Flags; + }; + + struct TEvent { + void* Data; + int Status; + ui16 Filter; + }; + using TEvents = TVector<TEvent>; - - virtual ~IPollerFace() { - } - + + virtual ~IPollerFace() { + } + void Set(void* ptr, SOCKET fd, ui16 flags) { - const TChange c = {fd, ptr, flags}; - - Set(c); - } - - virtual void Set(const TChange& change) = 0; - virtual void Wait(TEvents& events, TInstant deadLine) = 0; + const TChange c = {fd, ptr, flags}; + + Set(c); + } + + virtual void Set(const TChange& change) = 0; + virtual void Wait(TEvents& events, TInstant deadLine) = 0; virtual EContPoller PollEngine() const = 0; - + static THolder<IPollerFace> Default(); static THolder<IPollerFace> Construct(TStringBuf name); static THolder<IPollerFace> Construct(EContPoller poller); -}; +}; diff --git a/library/cpp/coroutine/engine/sockmap.h b/library/cpp/coroutine/engine/sockmap.h index fd189e1774..aa616da61e 100644 --- a/library/cpp/coroutine/engine/sockmap.h +++ b/library/cpp/coroutine/engine/sockmap.h @@ -1,24 +1,24 @@ -#pragma once - -#include <util/generic/hash.h> -#include <util/generic/vector.h> - -template <class T> -class TSocketMap { -public: +#pragma once + +#include <util/generic/hash.h> +#include <util/generic/vector.h> + +template <class T> +class TSocketMap { +public: T& Get(size_t idx) { - if (idx < 128000) { - if (V_.size() <= idx) { - V_.resize(idx + 1); - } - - return V_[idx]; - } - - return H_[idx]; - } - -private: + if (idx < 128000) { + if (V_.size() <= idx) { + V_.resize(idx + 1); + } + + return V_[idx]; + } + + return H_[idx]; + } + +private: TVector<T> V_; THashMap<size_t, T> H_; -}; +}; diff --git a/library/cpp/coroutine/engine/sockpool.cpp b/library/cpp/coroutine/engine/sockpool.cpp index b9482e780f..895dd686c7 100644 --- a/library/cpp/coroutine/engine/sockpool.cpp +++ b/library/cpp/coroutine/engine/sockpool.cpp @@ -13,7 +13,7 @@ void SetCommonSockOpts(SOCKET sock, const struct sockaddr* sa) { warn("bind"); } } else if (sa->sa_family == AF_INET6) { - sockaddr_in6 s_in6(*(const sockaddr_in6*)sa); + sockaddr_in6 s_in6(*(const sockaddr_in6*)sa); Zero(s_in6.sin6_addr); s_in6.sin6_port = 0; diff --git a/library/cpp/coroutine/engine/sockpool.h b/library/cpp/coroutine/engine/sockpool.h index 1ebb7e7b38..b34d5ace46 100644 --- a/library/cpp/coroutine/engine/sockpool.h +++ b/library/cpp/coroutine/engine/sockpool.h @@ -1,253 +1,253 @@ #pragma once - + #include "impl.h" #include "network.h" - + #include <util/network/address.h> #include <util/network/socket.h> #include <util/system/mutex.h> - + extern void SetCommonSockOpts(SOCKET sock, const struct sockaddr* sa = nullptr); - + class TSocketPool; class TPooledSocket { - class TImpl: public TIntrusiveListItem<TImpl>, public TSimpleRefCount<TImpl, TImpl> { - public: + class TImpl: public TIntrusiveListItem<TImpl>, public TSimpleRefCount<TImpl, TImpl> { + public: TImpl(SOCKET fd, TSocketPool* pool) noexcept - : Pool_(pool) - , IsKeepAlive_(false) - , Fd_(fd) - { - Touch(); - } - + : Pool_(pool) + , IsKeepAlive_(false) + , Fd_(fd) + { + Touch(); + } + static void Destroy(TImpl* impl) noexcept { - impl->DoDestroy(); - } - + impl->DoDestroy(); + } + void DoDestroy() noexcept { - if (!Closed() && IsKeepAlive() && IsInGoodState()) { - ReturnToPool(); - } else { - delete this; - } - } - + if (!Closed() && IsKeepAlive() && IsInGoodState()) { + ReturnToPool(); + } else { + delete this; + } + } + bool IsKeepAlive() const noexcept { - return IsKeepAlive_; - } - + return IsKeepAlive_; + } + void SetKeepAlive(bool ka) { - ::SetKeepAlive(Fd_, ka); - IsKeepAlive_ = ka; - } - + ::SetKeepAlive(Fd_, ka); + IsKeepAlive_ = ka; + } + SOCKET Socket() const noexcept { - return Fd_; - } - + return Fd_; + } + bool Closed() const noexcept { - return Fd_.Closed(); - } - + return Fd_.Closed(); + } + void Close() noexcept { - Fd_.Close(); - } - + Fd_.Close(); + } + bool IsInGoodState() const noexcept { - int err = 0; - socklen_t len = sizeof(err); - - getsockopt(Fd_, SOL_SOCKET, SO_ERROR, (char*)&err, &len); - - return !err; - } - + int err = 0; + socklen_t len = sizeof(err); + + getsockopt(Fd_, SOL_SOCKET, SO_ERROR, (char*)&err, &len); + + return !err; + } + bool IsOpen() const noexcept { return IsInGoodState() && IsNotSocketClosedByOtherSide(Fd_); - } - + } + void Touch() noexcept { - TouchTime_ = TInstant::Now(); - } - + TouchTime_ = TInstant::Now(); + } + const TInstant& LastTouch() const noexcept { - return TouchTime_; - } - - private: + return TouchTime_; + } + + private: inline void ReturnToPool() noexcept; - - private: - TSocketPool* Pool_; - bool IsKeepAlive_; - TSocketHolder Fd_; - TInstant TouchTime_; - }; - - friend class TSocketPool; - -public: + + private: + TSocketPool* Pool_; + bool IsKeepAlive_; + TSocketHolder Fd_; + TInstant TouchTime_; + }; + + friend class TSocketPool; + +public: TPooledSocket() : Impl_(nullptr) - { - } + { + } TPooledSocket(TImpl* impl) - : Impl_(impl) - { - } - + : Impl_(impl) + { + } + ~TPooledSocket() { - if (UncaughtException() && !!Impl_) { - Close(); + if (UncaughtException() && !!Impl_) { + Close(); } - } - + } + operator SOCKET() const noexcept { - return Impl_->Socket(); - } - + return Impl_->Socket(); + } + void SetKeepAlive(bool ka) { - Impl_->SetKeepAlive(ka); - } - + Impl_->SetKeepAlive(ka); + } + void Close() noexcept { - Impl_->Close(); - } - -private: - TIntrusivePtr<TImpl> Impl_; + Impl_->Close(); + } + +private: + TIntrusivePtr<TImpl> Impl_; }; struct TConnectData { TConnectData(TCont* cont, const TInstant& deadLine) - : Cont(cont) - , DeadLine(deadLine) - { - } - + : Cont(cont) + , DeadLine(deadLine) + { + } + TConnectData(TCont* cont, const TDuration& timeOut) - : Cont(cont) - , DeadLine(TInstant::Now() + timeOut) - { - } - - TCont* Cont; - const TInstant DeadLine; + : Cont(cont) + , DeadLine(TInstant::Now() + timeOut) + { + } + + TCont* Cont; + const TInstant DeadLine; }; class TSocketPool { - friend class TPooledSocket::TImpl; - -public: - typedef TAtomicSharedPtr<NAddr::IRemoteAddr> TAddrRef; + friend class TPooledSocket::TImpl; + +public: + typedef TAtomicSharedPtr<NAddr::IRemoteAddr> TAddrRef; TSocketPool(int ip, int port) - : Addr_(new NAddr::TIPv4Addr(TIpAddress((ui32)ip, (ui16)port))) - { - } - + : Addr_(new NAddr::TIPv4Addr(TIpAddress((ui32)ip, (ui16)port))) + { + } + TSocketPool(const TAddrRef& addr) - : Addr_(addr) - { - } + : Addr_(addr) + { + } void EraseStale(const TInstant& maxAge) noexcept { - TSockets toDelete; - - { - TGuard<TMutex> guard(Mutex_); - - for (TSockets::TIterator it = Pool_.Begin(); it != Pool_.End();) { - if (it->LastTouch() < maxAge) { - toDelete.PushBack(&*(it++)); - } else { - ++it; - } - } - } - } - + TSockets toDelete; + + { + TGuard<TMutex> guard(Mutex_); + + for (TSockets::TIterator it = Pool_.Begin(); it != Pool_.End();) { + if (it->LastTouch() < maxAge) { + toDelete.PushBack(&*(it++)); + } else { + ++it; + } + } + } + } + TPooledSocket Get(TConnectData* conn) { - TPooledSocket ret; - - if (TPooledSocket::TImpl* alive = GetImpl()) { - ret = TPooledSocket(alive); - } else { - ret = AllocateMore(conn); - } - - ret.Impl_->Touch(); - - return ret; - } - + TPooledSocket ret; + + if (TPooledSocket::TImpl* alive = GetImpl()) { + ret = TPooledSocket(alive); + } else { + ret = AllocateMore(conn); + } + + ret.Impl_->Touch(); + + return ret; + } + bool GetAlive(TPooledSocket& socket) { - if (TPooledSocket::TImpl* alive = GetImpl()) { - alive->Touch(); - socket = TPooledSocket(alive); - return true; + if (TPooledSocket::TImpl* alive = GetImpl()) { + alive->Touch(); + socket = TPooledSocket(alive); + return true; } - return false; - } + return false; + } -private: +private: TPooledSocket::TImpl* GetImpl() { - TGuard<TMutex> guard(Mutex_); - - while (!Pool_.Empty()) { - THolder<TPooledSocket::TImpl> ret(Pool_.PopFront()); - - if (ret->IsOpen()) { - return ret.Release(); + TGuard<TMutex> guard(Mutex_); + + while (!Pool_.Empty()) { + THolder<TPooledSocket::TImpl> ret(Pool_.PopFront()); + + if (ret->IsOpen()) { + return ret.Release(); } } return nullptr; - } - + } + void Release(TPooledSocket::TImpl* impl) noexcept { - TGuard<TMutex> guard(Mutex_); - - Pool_.PushFront(impl); - } - - TPooledSocket AllocateMore(TConnectData* conn); - -private: - TAddrRef Addr_; + TGuard<TMutex> guard(Mutex_); + + Pool_.PushFront(impl); + } + + TPooledSocket AllocateMore(TConnectData* conn); + +private: + TAddrRef Addr_; using TSockets = TIntrusiveListWithAutoDelete<TPooledSocket::TImpl, TDelete>; - TSockets Pool_; - TMutex Mutex_; + TSockets Pool_; + TMutex Mutex_; }; inline void TPooledSocket::TImpl::ReturnToPool() noexcept { - Pool_->Release(this); + Pool_->Release(this); } class TContIO: public IInputStream, public IOutputStream { -public: +public: TContIO(SOCKET fd, TCont* cont) - : Fd_(fd) - , Cont_(cont) - { - } - + : Fd_(fd) + , Cont_(cont) + { + } + void DoWrite(const void* buf, size_t len) override { NCoro::WriteI(Cont_, Fd_, buf, len).Checked(); - } - + } + size_t DoRead(void* buf, size_t len) override { return NCoro::ReadI(Cont_, Fd_, buf, len).Checked(); - } - + } + SOCKET Fd() const noexcept { - return Fd_; - } - -private: - SOCKET Fd_; - TCont* Cont_; + return Fd_; + } + +private: + SOCKET Fd_; + TCont* Cont_; }; diff --git a/library/cpp/coroutine/engine/trampoline.cpp b/library/cpp/coroutine/engine/trampoline.cpp index 10ea69ddc3..75254b58a8 100644 --- a/library/cpp/coroutine/engine/trampoline.cpp +++ b/library/cpp/coroutine/engine/trampoline.cpp @@ -1,16 +1,16 @@ #include "impl.h" #include "trampoline.h" - + #include "stack/stack_allocator.h" #include <util/system/info.h> #include <util/system/protect.h> #include <util/system/valgrind.h> #include <util/system/yassert.h> - + #include <cstdlib> #include <util/stream/format.h> - + namespace NCoro { @@ -30,7 +30,7 @@ TTrampoline::TTrampoline(NStack::IAllocator& allocator, ui32 stackSize, TFunc f, Func_(Cont_); } catch (...) {} } - + Cont_->Terminate(); } diff --git a/library/cpp/coroutine/engine/trampoline.h b/library/cpp/coroutine/engine/trampoline.h index 37b61cf015..23cc587c13 100644 --- a/library/cpp/coroutine/engine/trampoline.h +++ b/library/cpp/coroutine/engine/trampoline.h @@ -1,20 +1,20 @@ #pragma once - + #include "stack/stack_common.h" #include "stack/stack.h" #include <util/generic/noncopyable.h> #include <util/generic/ptr.h> -#include <util/system/context.h> -#include <util/system/defaults.h> - +#include <util/system/context.h> +#include <util/system/defaults.h> + #if !defined(STACK_GROW_DOWN) # error "unsupported" #endif -class TCont; +class TCont; typedef void (*TContFunc)(TCont*, void*); - + namespace NCoro { namespace NStack { @@ -31,9 +31,9 @@ namespace NCoro { TFunc f, TCont* cont ) noexcept; - + TArrayRef<char> Stack() noexcept; - + TExceptionSafeContext* Context() noexcept { return &Ctx_; } @@ -56,5 +56,5 @@ namespace NCoro { TExceptionSafeContext Ctx_; TFunc Func_; TCont* const Cont_; - }; -} + }; +} diff --git a/library/cpp/coroutine/engine/ya.make b/library/cpp/coroutine/engine/ya.make index 8c20b9afc3..de48968307 100644 --- a/library/cpp/coroutine/engine/ya.make +++ b/library/cpp/coroutine/engine/ya.make @@ -1,25 +1,25 @@ -LIBRARY() - +LIBRARY() + OWNER( pg g:balancer ) - + GENERATE_ENUM_SERIALIZATION(poller.h) GENERATE_ENUM_SERIALIZATION(stack/stack_common.h) -PEERDIR( +PEERDIR( contrib/libs/libc_compat library/cpp/containers/intrusive_rb_tree -) - -SRCS( +) + +SRCS( cont_poller.cpp helper.cpp - impl.cpp - iostatus.cpp + impl.cpp + iostatus.cpp network.cpp - poller.cpp + poller.cpp sockpool.cpp stack/stack.cpp stack/stack_allocator.cpp @@ -27,9 +27,9 @@ SRCS( stack/stack_storage.cpp stack/stack_utils.cpp trampoline.cpp -) - -END() +) + +END() RECURSE( stack/benchmark diff --git a/library/cpp/coroutine/listener/listen.cpp b/library/cpp/coroutine/listener/listen.cpp index 3d4e711d1d..13a3a9b3ae 100644 --- a/library/cpp/coroutine/listener/listen.cpp +++ b/library/cpp/coroutine/listener/listen.cpp @@ -1,15 +1,15 @@ -#include "listen.h" - +#include "listen.h" + #include <library/cpp/coroutine/engine/impl.h> #include <library/cpp/coroutine/engine/network.h> - -#include <util/network/ip.h> -#include <util/network/address.h> + +#include <util/network/ip.h> +#include <util/network/address.h> #include <util/generic/ylimits.h> -#include <util/generic/intrlist.h> - -using namespace NAddr; - +#include <util/generic/intrlist.h> + +using namespace NAddr; + namespace { union TSa { const sockaddr* Sa; @@ -24,10 +24,10 @@ namespace { inline bool operator==(const TSa& r) const noexcept { if (Sa->sa_family == r.Sa->sa_family) { switch (Sa->sa_family) { - case AF_INET: - return In->sin_port == r.In->sin_port && In->sin_addr.s_addr == r.In->sin_addr.s_addr; - case AF_INET6: - return In6->sin6_port == r.In6->sin6_port && !memcmp(&In6->sin6_addr, &r.In6->sin6_addr, sizeof(in6_addr)); + case AF_INET: + return In->sin_port == r.In->sin_port && In->sin_addr.s_addr == r.In->sin_addr.s_addr; + case AF_INET6: + return In6->sin6_port == r.In6->sin6_port && !memcmp(&In6->sin6_addr, &r.In6->sin6_addr, sizeof(in6_addr)); } } @@ -40,31 +40,31 @@ namespace { }; } -class TContListener::TImpl { +class TContListener::TImpl { private: struct TStoredAddrInfo: public TAddrInfo, private TNetworkAddress { inline TStoredAddrInfo(const struct addrinfo* ai, const TNetworkAddress& addr) noexcept - : TAddrInfo(ai) - , TNetworkAddress(addr) - { - } - }; - + : TAddrInfo(ai) + , TNetworkAddress(addr) + { + } + }; + private: - class TOneSocketListener: public TIntrusiveListItem<TOneSocketListener> { - public: - inline TOneSocketListener(TImpl* parent, IRemoteAddrPtr addr) - : Parent_(parent) + class TOneSocketListener: public TIntrusiveListItem<TOneSocketListener> { + public: + inline TOneSocketListener(TImpl* parent, IRemoteAddrPtr addr) + : Parent_(parent) , C_(nullptr) - , ListenSocket_(socket(addr->Addr()->sa_family, SOCK_STREAM, 0)) + , ListenSocket_(socket(addr->Addr()->sa_family, SOCK_STREAM, 0)) , Addr_(std::move(addr)) - { - if (ListenSocket_ == INVALID_SOCKET) { - ythrow TSystemError() << "can not create socket"; - } - - FixIPv6ListenSocket(ListenSocket_); - CheckedSetSockOpt(ListenSocket_, SOL_SOCKET, SO_REUSEADDR, 1, "reuse addr"); + { + if (ListenSocket_ == INVALID_SOCKET) { + ythrow TSystemError() << "can not create socket"; + } + + FixIPv6ListenSocket(ListenSocket_); + CheckedSetSockOpt(ListenSocket_, SOL_SOCKET, SO_REUSEADDR, 1, "reuse addr"); const TOptions& opts = Parent_->Opts_; if (opts.SendBufSize) { @@ -77,106 +77,106 @@ private: SetReusePort(ListenSocket_, opts.ReusePort); } - SetNonBlock(ListenSocket_); - - if (bind(ListenSocket_, Addr_->Addr(), Addr_->Len()) < 0) { - ythrow TSystemError() << "bind failed"; - } - } - + SetNonBlock(ListenSocket_); + + if (bind(ListenSocket_, Addr_->Addr(), Addr_->Len()) < 0) { + ythrow TSystemError() << "bind failed"; + } + } + inline ~TOneSocketListener() { - Stop(); - } - + Stop(); + } + public: inline void Run(TCont* cont) noexcept { C_ = cont; DoRun(); C_ = nullptr; - } - - inline void StartListen() { - if (!C_) { - const TOptions& opts = Parent_->Opts_; - - if (listen(ListenSocket_, (int)Min<size_t>(Max<int>(), opts.ListenQueue)) < 0) { - ythrow TSystemError() << "listen failed"; - } - - if (opts.EnableDeferAccept) { - SetDeferAccept(ListenSocket_); - } - + } + + inline void StartListen() { + if (!C_) { + const TOptions& opts = Parent_->Opts_; + + if (listen(ListenSocket_, (int)Min<size_t>(Max<int>(), opts.ListenQueue)) < 0) { + ythrow TSystemError() << "listen failed"; + } + + if (opts.EnableDeferAccept) { + SetDeferAccept(ListenSocket_); + } + C_ = Parent_->E_->Create<TOneSocketListener, &TOneSocketListener::Run>(this, "listen_job"); - } - } + } + } inline const IRemoteAddr* Addr() const noexcept { - return Addr_.Get(); - } - + return Addr_.Get(); + } + inline void Stop() noexcept { - if (C_) { - C_->Cancel(); - - while (C_) { + if (C_) { + C_->Cancel(); + + while (C_) { Parent_->E_->Running()->Yield(); - } - } - } - - private: + } + } + } + + private: inline void DoRun() noexcept { while (!C_->Cancelled()) { - try { - TOpaqueAddr remote; + try { + TOpaqueAddr remote; const int res = NCoro::AcceptI(C_, ListenSocket_, remote.MutableAddr(), remote.LenPtr()); - - if (res < 0) { - const int err = -res; - - if (err != ECONNABORTED) { - if (err == ECANCELED) { - break; - } + + if (res < 0) { + const int err = -res; + + if (err != ECONNABORTED) { + if (err == ECANCELED) { + break; + } if (errno == EMFILE) { C_->SleepT(TDuration::MilliSeconds(1)); } - - ythrow TSystemError(err) << "can not accept"; - } - } else { - TSocketHolder c((SOCKET)res); - + + ythrow TSystemError(err) << "can not accept"; + } + } else { + TSocketHolder c((SOCKET)res); + const ICallBack::TAcceptFull acc = { - &c, - &remote, - Addr(), - }; - + &c, + &remote, + Addr(), + }; + Parent_->Cb_->OnAcceptFull(acc); - } - } catch (...) { - try { - Parent_->Cb_->OnError(); - } catch (...) { - } - } - } - + } + } catch (...) { + try { + Parent_->Cb_->OnError(); + } catch (...) { + } + } + } + try { Parent_->Cb_->OnStop(&ListenSocket_); } catch (...) { } - } - - private: + } + + private: const TImpl* const Parent_; - TCont* C_; - TSocketHolder ListenSocket_; + TCont* C_; + TSocketHolder ListenSocket_; const IRemoteAddrPtr Addr_; - }; - + }; + private: class TListeners: public TIntrusiveListWithAutoDelete<TOneSocketListener, TDelete> { private: @@ -184,11 +184,11 @@ private: using TIt = std::conditional_t<std::is_const<T>::value, typename T::TConstIterator, typename T::TIterator>; template <class T> - static inline TIt<T> FindImpl(T* t, const IRemoteAddr& addr) { + static inline TIt<T> FindImpl(T* t, const IRemoteAddr& addr) { const TSa sa(addr.Addr()); - TIt<T> it = t->Begin(); - TIt<T> const end = t->End(); + TIt<T> it = t->Begin(); + TIt<T> const end = t->End(); while (it != end && sa != it->Addr()->Addr()) { ++it; @@ -207,20 +207,20 @@ private: } }; -public: +public: inline TImpl(ICallBack* cb, TContExecutor* e, const TOptions& opts) noexcept - : E_(e) - , Cb_(cb) - , Opts_(opts) - { - } - - inline void Listen() { - for (TListeners::TIterator it = L_.Begin(); it != L_.End(); ++it) { - it->StartListen(); - } - } - + : E_(e) + , Cb_(cb) + , Opts_(opts) + { + } + + inline void Listen() { + for (TListeners::TIterator it = L_.Begin(); it != L_.End(); ++it) { + it->StartListen(); + } + } + inline void Listen(const IRemoteAddr& addr) { const TListeners::TIterator it = L_.Find(addr); @@ -233,62 +233,62 @@ public: const TSa sa(addr.Addr()); switch (sa.Sa->sa_family) { - case AF_INET: + case AF_INET: L_.PushBack(new TOneSocketListener(this, MakeHolder<TIPv4Addr>(*sa.In))); - break; - case AF_INET6: + break; + case AF_INET6: L_.PushBack(new TOneSocketListener(this, MakeHolder<TIPv6Addr>(*sa.In6))); - break; - default: + break; + default: ythrow yexception() << TStringBuf("unknown protocol"); } } - inline void Bind(const TIpAddress& addr) { + inline void Bind(const TIpAddress& addr) { L_.PushBack(new TOneSocketListener(this, MakeHolder<TIPv4Addr>(addr))); - } - - inline void Bind(const TNetworkAddress& addr) { - for (TNetworkAddress::TIterator it = addr.Begin(); it != addr.End(); ++it) { + } + + inline void Bind(const TNetworkAddress& addr) { + for (TNetworkAddress::TIterator it = addr.Begin(); it != addr.End(); ++it) { L_.PushBack(new TOneSocketListener(this, MakeHolder<TStoredAddrInfo>(&*it, addr))); - } - } - + } + } + inline void StopListenAddr(const IRemoteAddr& addr) { const TListeners::TIterator it = L_.Find(addr); - + if (it != L_.End()) { delete &*it; } - } + } -private: +private: TContExecutor* const E_; ICallBack* const Cb_; - TListeners L_; - const TOptions Opts_; -}; - -TContListener::TContListener(ICallBack* cb, TContExecutor* e, const TOptions& opts) - : Impl_(new TImpl(cb, e, opts)) -{ -} - + TListeners L_; + const TOptions Opts_; +}; + +TContListener::TContListener(ICallBack* cb, TContExecutor* e, const TOptions& opts) + : Impl_(new TImpl(cb, e, opts)) +{ +} + TContListener::~TContListener() { -} - +} + namespace { template <class T> static inline T&& CheckImpl(T&& impl) { Y_ENSURE_EX(impl, yexception() << "not running"); return std::forward<T>(impl); - } + } } - + void TContListener::Listen(const IRemoteAddr& addr) { CheckImpl(Impl_)->Listen(addr); -} - +} + void TContListener::Listen(const TIpAddress& addr) { return Listen(TIPv4Addr(addr)); } @@ -296,28 +296,28 @@ void TContListener::Listen(const TIpAddress& addr) { void TContListener::Listen(const TNetworkAddress& addr) { for (TNetworkAddress::TIterator it = addr.Begin(); it != addr.End(); ++it) { Listen(TAddrInfo(&*it)); - } + } } - + void TContListener::Listen() { CheckImpl(Impl_)->Listen(); } void TContListener::Bind(const IRemoteAddr& addr) { CheckImpl(Impl_)->Bind(addr); -} - +} + void TContListener::Bind(const TIpAddress& addr) { return Bind(TIPv4Addr(addr)); } - + void TContListener::Bind(const TNetworkAddress& addr) { CheckImpl(Impl_)->Bind(addr); -} - +} + void TContListener::Stop() noexcept { - Impl_.Destroy(); -} + Impl_.Destroy(); +} void TContListener::StopListenAddr(const IRemoteAddr& addr) { CheckImpl(Impl_)->StopListenAddr(addr); @@ -326,13 +326,13 @@ void TContListener::StopListenAddr(const IRemoteAddr& addr) { void TContListener::StopListenAddr(const TIpAddress& addr) { return StopListenAddr(TIPv4Addr(addr)); } - + void TContListener::StopListenAddr(const TNetworkAddress& addr) { for (TNetworkAddress::TIterator it = addr.Begin(); it != addr.End(); ++it) { StopListenAddr(TAddrInfo(&*it)); } -} - +} + void TContListener::ICallBack::OnAcceptFull(const TAcceptFull& params) { const TSa remote(params.Remote->Addr()); const TSa local(params.Local->Addr()); diff --git a/library/cpp/coroutine/listener/listen.h b/library/cpp/coroutine/listener/listen.h index 3a89cd3ecc..a61bdeb6df 100644 --- a/library/cpp/coroutine/listener/listen.h +++ b/library/cpp/coroutine/listener/listen.h @@ -1,41 +1,41 @@ #pragma once - + #include <util/generic/ptr.h> -#include <util/generic/ylimits.h> - +#include <util/generic/ylimits.h> + struct TIpAddress; -class TContExecutor; -class TSocketHolder; -class TNetworkAddress; - -namespace NAddr { - class IRemoteAddr; -} - -class TContListener { -public: - struct TOptions { +class TContExecutor; +class TSocketHolder; +class TNetworkAddress; + +namespace NAddr { + class IRemoteAddr; +} + +class TContListener { +public: + struct TOptions { inline TOptions() noexcept - : ListenQueue(Max<size_t>()) + : ListenQueue(Max<size_t>()) , SendBufSize(0) , RecvBufSize(0) - , EnableDeferAccept(false) + , EnableDeferAccept(false) , ReusePort(false) - { - } - + { + } + inline TOptions& SetListenQueue(size_t len) noexcept { - ListenQueue = len; - - return *this; - } - + ListenQueue = len; + + return *this; + } + inline TOptions& SetDeferAccept(bool enable) noexcept { - EnableDeferAccept = enable; - - return *this; - } - + EnableDeferAccept = enable; + + return *this; + } + inline TOptions& SetSendBufSize(unsigned size) noexcept { SendBufSize = size; @@ -54,61 +54,61 @@ public: return *this; } - size_t ListenQueue; + size_t ListenQueue; unsigned SendBufSize; unsigned RecvBufSize; - bool EnableDeferAccept; + bool EnableDeferAccept; bool ReusePort; - }; - - class ICallBack { - public: - struct TAccept { - TSocketHolder* S; + }; + + class ICallBack { + public: + struct TAccept { + TSocketHolder* S; const TIpAddress* Remote; const TIpAddress* Local; }; struct TAcceptFull { TSocketHolder* S; - const NAddr::IRemoteAddr* Remote; - const NAddr::IRemoteAddr* Local; - }; + const NAddr::IRemoteAddr* Remote; + const NAddr::IRemoteAddr* Local; + }; virtual void OnAccept(const TAccept&) { } - + virtual void OnAcceptFull(const TAcceptFull&); - /* - * will be called from catch (...) {} context - * so your can re-throw current exception and work around it - */ - virtual void OnError() = 0; - + /* + * will be called from catch (...) {} context + * so your can re-throw current exception and work around it + */ + virtual void OnError() = 0; + virtual void OnStop(TSocketHolder*); - virtual ~ICallBack() { - } - }; - - TContListener(ICallBack* cb, TContExecutor* e, const TOptions& opts = TOptions()); + virtual ~ICallBack() { + } + }; + + TContListener(ICallBack* cb, TContExecutor* e, const TOptions& opts = TOptions()); ~TContListener(); - - /// start listener threads - void Listen(); + + /// start listener threads + void Listen(); void Listen(const NAddr::IRemoteAddr& addr); void Listen(const TIpAddress& addr); void Listen(const TNetworkAddress& addr); - /// bind server on address. Can be called multiple times to bind on more then one address + /// bind server on address. Can be called multiple times to bind on more then one address void Bind(const NAddr::IRemoteAddr& addr); - void Bind(const TIpAddress& addr); - void Bind(const TNetworkAddress& addr); + void Bind(const TIpAddress& addr); + void Bind(const TNetworkAddress& addr); void Stop() noexcept; - + void StopListenAddr(const NAddr::IRemoteAddr& addr); void StopListenAddr(const TIpAddress& addr); void StopListenAddr(const TNetworkAddress& addr); @@ -119,7 +119,7 @@ public: Listen(addr); } -private: - class TImpl; - THolder<TImpl> Impl_; -}; +private: + class TImpl; + THolder<TImpl> Impl_; +}; diff --git a/library/cpp/coroutine/listener/ya.make b/library/cpp/coroutine/listener/ya.make index 700c3abe3e..9544e24c41 100644 --- a/library/cpp/coroutine/listener/ya.make +++ b/library/cpp/coroutine/listener/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - -OWNER(pg) - -PEERDIR( +LIBRARY() + +OWNER(pg) + +PEERDIR( library/cpp/coroutine/engine -) - -SRCS( - listen.cpp -) - -END() +) + +SRCS( + listen.cpp +) + +END() diff --git a/library/cpp/coroutine/ya.make b/library/cpp/coroutine/ya.make index 34e30f2b25..2f2d57289b 100644 --- a/library/cpp/coroutine/ya.make +++ b/library/cpp/coroutine/ya.make @@ -1,11 +1,11 @@ -RECURSE( - benchmark - dns - dns/example - engine +RECURSE( + benchmark + dns + dns/example + engine engine/stack/ut - listener - test + listener + test ut util -) +) diff --git a/library/cpp/dbg_output/DONT_COMMIT.h b/library/cpp/dbg_output/DONT_COMMIT.h index e7b3182c20..4a106d5cfb 100644 --- a/library/cpp/dbg_output/DONT_COMMIT.h +++ b/library/cpp/dbg_output/DONT_COMMIT.h @@ -4,13 +4,13 @@ // The latter is allowed only locally, so this file is named // in such a way that including it prevents from committing the #include via ARC-1205. -#define DBGDUMP_INLINE_IF_INCLUDED inline - -#include "dump.cpp" -#include "dumpers.cpp" -#include "engine.cpp" - +#define DBGDUMP_INLINE_IF_INCLUDED inline + +#include "dump.cpp" +#include "dumpers.cpp" +#include "engine.cpp" + #include <library/cpp/colorizer/colors.cpp> #include <library/cpp/colorizer/output.cpp> -#undef DBGDUMP_INLINE_IF_INCLUDED +#undef DBGDUMP_INLINE_IF_INCLUDED diff --git a/library/cpp/dbg_output/auto.h b/library/cpp/dbg_output/auto.h index 8d96167f6a..99961bfa6b 100644 --- a/library/cpp/dbg_output/auto.h +++ b/library/cpp/dbg_output/auto.h @@ -10,13 +10,13 @@ // Usage: struct TMyStruct { int A, B; }; DEFINE_DUMPER(TMyStruct, A, B); Cout << TMyStruct{3, 4}; // yields {"A": 3, "B": 4} -#define DEFINE_DUMPER(C, ...) \ - template <> \ - struct TDumper<C> { \ - template <class S> \ - static inline void Dump(S& s, const C& v) { \ +#define DEFINE_DUMPER(C, ...) \ + template <> \ + struct TDumper<C> { \ + template <class S> \ + static inline void Dump(S& s, const C& v) { \ s << DumpRaw("{") Y_PASS_VA_ARGS(Y_MAP_ARGS_WITH_LAST(__DEFINE_DUMPER_NONLAST__, __DEFINE_DUMPER_IMPL__, __VA_ARGS__)) << DumpRaw("}"); \ - } \ + } \ }; #define __DEFINE_DUMPER_IMPL__(x) << DumpRaw("\"" #x "\": ") << v.x #define __DEFINE_DUMPER_NONLAST__(x) __DEFINE_DUMPER_IMPL__(x) << DumpRaw(", ") diff --git a/library/cpp/dbg_output/colorscheme.h b/library/cpp/dbg_output/colorscheme.h index a5b9cf749a..49c4a33a8a 100644 --- a/library/cpp/dbg_output/colorscheme.h +++ b/library/cpp/dbg_output/colorscheme.h @@ -4,93 +4,93 @@ #include <library/cpp/colorizer/output.h> #ifndef DBG_OUTPUT_DEFAULT_COLOR_SCHEME -#define DBG_OUTPUT_DEFAULT_COLOR_SCHEME NDbgDump::NColorScheme::TPlain +#define DBG_OUTPUT_DEFAULT_COLOR_SCHEME NDbgDump::NColorScheme::TPlain #endif #define DBG_OUTPUT_COLOR_HANDLER(NAME) \ - template <class S> \ + template <class S> \ inline void NAME(S& stream) namespace NDbgDump { - namespace NColorScheme { - /// Start by copying this one if you want to define a custom color scheme. - struct TPlain { - // Foreground color modifiers - DBG_OUTPUT_COLOR_HANDLER(Markup) { + namespace NColorScheme { + /// Start by copying this one if you want to define a custom color scheme. + struct TPlain { + // Foreground color modifiers + DBG_OUTPUT_COLOR_HANDLER(Markup) { Y_UNUSED(stream); - } - DBG_OUTPUT_COLOR_HANDLER(String) { + } + DBG_OUTPUT_COLOR_HANDLER(String) { Y_UNUSED(stream); - } - DBG_OUTPUT_COLOR_HANDLER(Literal) { + } + DBG_OUTPUT_COLOR_HANDLER(Literal) { Y_UNUSED(stream); - } - DBG_OUTPUT_COLOR_HANDLER(ResetType) { + } + DBG_OUTPUT_COLOR_HANDLER(ResetType) { Y_UNUSED(stream); - } + } - // Background color modifiers - DBG_OUTPUT_COLOR_HANDLER(Key) { + // Background color modifiers + DBG_OUTPUT_COLOR_HANDLER(Key) { Y_UNUSED(stream); - } - DBG_OUTPUT_COLOR_HANDLER(Value) { + } + DBG_OUTPUT_COLOR_HANDLER(Value) { Y_UNUSED(stream); - } - DBG_OUTPUT_COLOR_HANDLER(ResetRole) { + } + DBG_OUTPUT_COLOR_HANDLER(ResetRole) { Y_UNUSED(stream); - } - }; + } + }; - /// Use this one if you want colors but are lazy enough to define a custom color scheme. - /// Be careful enough to use DumpRaw for avoiding an endless recursion. - /// Enforce controls whether colors should be applied even if stdout is not a TTY. - template <bool Enforce = false> - class TEyebleed { - public: - TEyebleed() { - if (Enforce) { - Colors.Enable(); - } + /// Use this one if you want colors but are lazy enough to define a custom color scheme. + /// Be careful enough to use DumpRaw for avoiding an endless recursion. + /// Enforce controls whether colors should be applied even if stdout is not a TTY. + template <bool Enforce = false> + class TEyebleed { + public: + TEyebleed() { + if (Enforce) { + Colors.Enable(); + } } - // Foreground color modifiers - DBG_OUTPUT_COLOR_HANDLER(Markup) { - stream << DumpRaw(Colors.LightGreenColor()); - } - DBG_OUTPUT_COLOR_HANDLER(String) { - stream << DumpRaw(Colors.YellowColor()); - } - DBG_OUTPUT_COLOR_HANDLER(Literal) { - stream << DumpRaw(Colors.LightRedColor()); - } - DBG_OUTPUT_COLOR_HANDLER(ResetType) { - stream << DumpRaw(Colors.OldColor()); - } + // Foreground color modifiers + DBG_OUTPUT_COLOR_HANDLER(Markup) { + stream << DumpRaw(Colors.LightGreenColor()); + } + DBG_OUTPUT_COLOR_HANDLER(String) { + stream << DumpRaw(Colors.YellowColor()); + } + DBG_OUTPUT_COLOR_HANDLER(Literal) { + stream << DumpRaw(Colors.LightRedColor()); + } + DBG_OUTPUT_COLOR_HANDLER(ResetType) { + stream << DumpRaw(Colors.OldColor()); + } - // Background color modifiers + // Background color modifiers // TODO: support backgrounds in library/cpp/colorizer - DBG_OUTPUT_COLOR_HANDLER(Key) { - if (Depth++ == 0 && Colors.IsTTY()) { + DBG_OUTPUT_COLOR_HANDLER(Key) { + if (Depth++ == 0 && Colors.IsTTY()) { stream << DumpRaw(TStringBuf("\033[42m")); - } + } } - DBG_OUTPUT_COLOR_HANDLER(Value) { - if (Depth++ == 0 && Colors.IsTTY()) { + DBG_OUTPUT_COLOR_HANDLER(Value) { + if (Depth++ == 0 && Colors.IsTTY()) { stream << DumpRaw(TStringBuf("\033[44m")); - } + } } - DBG_OUTPUT_COLOR_HANDLER(ResetRole) { - if (--Depth == 0 && Colors.IsTTY()) { + DBG_OUTPUT_COLOR_HANDLER(ResetRole) { + if (--Depth == 0 && Colors.IsTTY()) { stream << DumpRaw(TStringBuf("\033[49m")); - } + } } - private: - NColorizer::TColors Colors; - size_t Depth = 0; - }; - } -} + private: + NColorizer::TColors Colors; + size_t Depth = 0; + }; + } +} namespace NPrivate { template <typename CS> diff --git a/library/cpp/dbg_output/dump.cpp b/library/cpp/dbg_output/dump.cpp index ab09a93822..c3e3947008 100644 --- a/library/cpp/dbg_output/dump.cpp +++ b/library/cpp/dbg_output/dump.cpp @@ -1 +1 @@ -#include "dump.h" +#include "dump.h" diff --git a/library/cpp/dbg_output/dump.h b/library/cpp/dbg_output/dump.h index c7efa105ee..184df606c1 100644 --- a/library/cpp/dbg_output/dump.h +++ b/library/cpp/dbg_output/dump.h @@ -1,106 +1,106 @@ -#pragma once - -#include "engine.h" -#include "dumpers.h" +#pragma once + +#include "engine.h" +#include "dumpers.h" #include "auto.h" #include "colorscheme.h" - -#include <util/stream/format.h> + +#include <util/stream/format.h> #include <util/system/type_name.h> -#include <util/generic/hash_set.h> +#include <util/generic/hash_set.h> #include <utility> - -/* - * Cout << DbgDump(any) << Endl; - * Cout << DbgDumpDeep(any) << Endl; - * Cout << DbgDump(any).SetIndent(true) << Endl; - * - * specialize TDumper<your type> for extending dumper - */ - -namespace NPrivate { - template <class TColorScheme> - struct TTraitsShallow { + +/* + * Cout << DbgDump(any) << Endl; + * Cout << DbgDumpDeep(any) << Endl; + * Cout << DbgDump(any).SetIndent(true) << Endl; + * + * specialize TDumper<your type> for extending dumper + */ + +namespace NPrivate { + template <class TColorScheme> + struct TTraitsShallow { struct TDump: public TDumpBase, public TColorSchemeContainer<TColorScheme> { - template <typename... Args> - inline TDump(Args&&... args) + template <typename... Args> + inline TDump(Args&&... args) : TDumpBase(std::forward<Args>(args)...) - { - } - - template <class V> - inline void Pointer(const V* v) { - if (v) { + { + } + + template <class V> + inline void Pointer(const V* v) { + if (v) { *this << DumpRaw("(") << DumpRaw(TypeName(v).data()) << DumpRaw(")") << Hex((size_t)v); - } else { + } else { *this << DumpRaw("(") << DumpRaw(TypeName<V>().data()) << DumpRaw("*)nullptr"); - } - } - }; - }; - - template <class TColorScheme> - struct TTraitsDeep { + } + } + }; + }; + + template <class TColorScheme> + struct TTraitsDeep { struct TDump: public TDumpBase, public TColorSchemeContainer<TColorScheme> { - template <typename... Args> - inline TDump(Args&&... args) + template <typename... Args> + inline TDump(Args&&... args) : TDumpBase(std::forward<Args>(args)...) - { - } - - template <class V> - inline void Pointer(const V* v) { + { + } + + template <class V> + inline void Pointer(const V* v) { if (v && !Visited.contains((size_t)v)) { - Visited.insert((size_t)v); + Visited.insert((size_t)v); *this << DumpRaw("(") << DumpRaw(TypeName(v).data()) << DumpRaw(")") << Hex((size_t)v) << DumpRaw(" -> ") << *v; - Visited.erase((size_t)v); - } else { + Visited.erase((size_t)v); + } else { *this << DumpRaw("(") << DumpRaw(TypeName<V>().data()) << DumpRaw("*)nullptr"); - } - } - + } + } + THashSet<size_t> Visited; - }; - }; - - template <class T, class TTraits> - struct TDbgDump { - inline TDbgDump(const T* t) - : T_(t) - , Indent(false) - { - } - + }; + }; + + template <class T, class TTraits> + struct TDbgDump { + inline TDbgDump(const T* t) + : T_(t) + , Indent(false) + { + } + inline void DumpTo(IOutputStream& out) const { - typename TTraits::TDump d(out, Indent); - - d << *T_; - } - + typename TTraits::TDump d(out, Indent); + + d << *T_; + } + inline TDbgDump& SetIndent(bool v) noexcept { - Indent = v; - - return *this; - } - - const T* T_; - bool Indent; - }; - - template <class T, class TTraits> + Indent = v; + + return *this; + } + + const T* T_; + bool Indent; + }; + + template <class T, class TTraits> static inline IOutputStream& operator<<(IOutputStream& out, const TDbgDump<T, TTraits>& d) { - d.DumpTo(out); - - return out; - } -} - + d.DumpTo(out); + + return out; + } +} + template <class T, class TColorScheme = DBG_OUTPUT_DEFAULT_COLOR_SCHEME> static inline ::NPrivate::TDbgDump<T, ::NPrivate::TTraitsShallow<TColorScheme>> DbgDump(const T& t) { return {std::addressof(t)}; -} - +} + template <class T, class TColorScheme = DBG_OUTPUT_DEFAULT_COLOR_SCHEME> static inline ::NPrivate::TDbgDump<T, ::NPrivate::TTraitsDeep<TColorScheme>> DbgDumpDeep(const T& t) { return {std::addressof(t)}; -} +} diff --git a/library/cpp/dbg_output/dumpers.cpp b/library/cpp/dbg_output/dumpers.cpp index b85d15755b..0f1775ef37 100644 --- a/library/cpp/dbg_output/dumpers.cpp +++ b/library/cpp/dbg_output/dumpers.cpp @@ -1 +1 @@ -#include "dumpers.h" +#include "dumpers.h" diff --git a/library/cpp/dbg_output/dumpers.h b/library/cpp/dbg_output/dumpers.h index 4868e97da0..cb4ea68126 100644 --- a/library/cpp/dbg_output/dumpers.h +++ b/library/cpp/dbg_output/dumpers.h @@ -1,52 +1,52 @@ -#pragma once - -#include "engine.h" - -#include <util/generic/fwd.h> -#include <util/generic/strbuf.h> +#pragma once + +#include "engine.h" + +#include <util/generic/fwd.h> +#include <util/generic/strbuf.h> #include <util/generic/string.h> - -//smart pointers -template <class T, class D> -struct TDumper<TAutoPtr<T, D>> { - template <class S> - static inline void Dump(S& s, const TAutoPtr<T, D>& v) { - s << DumpRaw("TAutoPtr(") << v.Get() << DumpRaw(")"); - } -}; - -template <class T, class D> -struct TDumper<THolder<T, D>> { - template <class S> - static inline void Dump(S& s, const THolder<T, D>& v) { - s << DumpRaw("THolder(") << v.Get() << DumpRaw(")"); - } -}; - -template <class T, class Ops> -struct TDumper<TIntrusivePtr<T, Ops>> { - template <class S> - static inline void Dump(S& s, const TIntrusivePtr<T, Ops>& v) { - s << DumpRaw("TIntrusivePtr(") << v.Get() << DumpRaw(")"); - } -}; - -template <class T, class C, class D> -struct TDumper<TSharedPtr<T, C, D>> { - template <class S> - static inline void Dump(S& s, const TSharedPtr<T, C, D>& v) { - s << DumpRaw("TSharedPtr(") << v.Get() << DumpRaw(")"); - } -}; - -template <class T, class C, class D> -struct TDumper<TCopyPtr<T, C, D>> { - template <class S> - static inline void Dump(S& s, const TCopyPtr<T, C, D>& v) { - s << DumpRaw("TCopyPtr(") << v.Get() << DumpRaw(")"); - } -}; - + +//smart pointers +template <class T, class D> +struct TDumper<TAutoPtr<T, D>> { + template <class S> + static inline void Dump(S& s, const TAutoPtr<T, D>& v) { + s << DumpRaw("TAutoPtr(") << v.Get() << DumpRaw(")"); + } +}; + +template <class T, class D> +struct TDumper<THolder<T, D>> { + template <class S> + static inline void Dump(S& s, const THolder<T, D>& v) { + s << DumpRaw("THolder(") << v.Get() << DumpRaw(")"); + } +}; + +template <class T, class Ops> +struct TDumper<TIntrusivePtr<T, Ops>> { + template <class S> + static inline void Dump(S& s, const TIntrusivePtr<T, Ops>& v) { + s << DumpRaw("TIntrusivePtr(") << v.Get() << DumpRaw(")"); + } +}; + +template <class T, class C, class D> +struct TDumper<TSharedPtr<T, C, D>> { + template <class S> + static inline void Dump(S& s, const TSharedPtr<T, C, D>& v) { + s << DumpRaw("TSharedPtr(") << v.Get() << DumpRaw(")"); + } +}; + +template <class T, class C, class D> +struct TDumper<TCopyPtr<T, C, D>> { + template <class S> + static inline void Dump(S& s, const TCopyPtr<T, C, D>& v) { + s << DumpRaw("TCopyPtr(") << v.Get() << DumpRaw(")"); + } +}; + //small ints // Default dumper prints them via IOutputStream << (value), which results in raw // chars, not integer values. Cast to a bigger int type to force printing as @@ -60,20 +60,20 @@ template <> struct TDumper<i8>: public TDumper<i32> { }; -//chars -template <> -struct TDumper<char>: public TCharDumper { -}; - -template <> -struct TDumper<wchar16>: public TCharDumper { -}; - -//pairs -template <class A, class B> -struct TDumper<std::pair<A, B>> { - template <class S> - static inline void Dump(S& s, const std::pair<A, B>& v) { +//chars +template <> +struct TDumper<char>: public TCharDumper { +}; + +template <> +struct TDumper<wchar16>: public TCharDumper { +}; + +//pairs +template <class A, class B> +struct TDumper<std::pair<A, B>> { + template <class S> + static inline void Dump(S& s, const std::pair<A, B>& v) { s.ColorScheme.Key(s); s.ColorScheme.Literal(s); s << v.first; @@ -86,18 +86,18 @@ struct TDumper<std::pair<A, B>> { s << v.second; s.ColorScheme.ResetType(s); s.ColorScheme.ResetRole(s); - } -}; - -//sequences -template <class T, class A> + } +}; + +//sequences +template <class T, class A> struct TDumper<TVector<T, A>>: public TSeqDumper { -}; - -template <class T, class A> -struct TDumper<std::vector<T, A>>: public TSeqDumper { -}; - +}; + +template <class T, class A> +struct TDumper<std::vector<T, A>>: public TSeqDumper { +}; + template <class T> struct TDumper<TArrayRef<T>>: public TSeqDumper { }; @@ -106,68 +106,68 @@ template <class T, size_t N> struct TDumper<std::array<T, N>>: public TSeqDumper { }; -template <class T, class A> +template <class T, class A> struct TDumper<TDeque<T, A>>: public TSeqDumper { -}; - -template <class T, class A> +}; + +template <class T, class A> struct TDumper<TList<T, A>>: public TSeqDumper { -}; - -//associatives -template <class K, class V, class P, class A> +}; + +//associatives +template <class K, class V, class P, class A> struct TDumper<TMap<K, V, P, A>>: public TAssocDumper { -}; - -template <class K, class V, class P, class A> +}; + +template <class K, class V, class P, class A> struct TDumper<TMultiMap<K, V, P, A>>: public TAssocDumper { -}; - -template <class T, class P, class A> +}; + +template <class T, class P, class A> struct TDumper<TSet<T, P, A>>: public TAssocDumper { -}; - -template <class T, class P, class A> +}; + +template <class T, class P, class A> struct TDumper<TMultiSet<T, P, A>>: public TAssocDumper { -}; - -template <class K, class V, class H, class P, class A> +}; + +template <class K, class V, class H, class P, class A> struct TDumper<THashMap<K, V, H, P, A>>: public TAssocDumper { -}; - -template <class K, class V, class H, class P, class A> +}; + +template <class K, class V, class H, class P, class A> struct TDumper<THashMultiMap<K, V, H, P, A>>: public TAssocDumper { -}; - -template <class T, class H, class P, class A> +}; + +template <class T, class H, class P, class A> struct TDumper<THashSet<T, H, P, A>>: public TAssocDumper { -}; - -template <class T, class H, class P, class A> +}; + +template <class T, class H, class P, class A> struct TDumper<THashMultiSet<T, H, P, A>>: public TAssocDumper { -}; - -//strings -template <> +}; + +//strings +template <> struct TDumper<TString>: public TStrDumper { -}; - -template <> -struct TDumper<const char*>: public TStrDumper { -}; - -template <> +}; + +template <> +struct TDumper<const char*>: public TStrDumper { +}; + +template <> struct TDumper<TUtf16String>: public TStrDumper { -}; - -template <> -struct TDumper<const wchar16*>: public TStrDumper { -}; - -template <class C, class T, class A> -struct TDumper<std::basic_string<C, T, A>>: public TStrDumper { -}; - -template <class TChar> +}; + +template <> +struct TDumper<const wchar16*>: public TStrDumper { +}; + +template <class C, class T, class A> +struct TDumper<std::basic_string<C, T, A>>: public TStrDumper { +}; + +template <class TChar> struct TDumper<TBasicStringBuf<TChar>>: public TStrDumper { -}; +}; diff --git a/library/cpp/dbg_output/engine.cpp b/library/cpp/dbg_output/engine.cpp index dcb9f02522..3b6003bb1a 100644 --- a/library/cpp/dbg_output/engine.cpp +++ b/library/cpp/dbg_output/engine.cpp @@ -1,33 +1,33 @@ -#include "engine.h" - -#include <util/string/cast.h> -#include <util/string/escape.h> - -#if !defined(DBGDUMP_INLINE_IF_INCLUDED) -#define DBGDUMP_INLINE_IF_INCLUDED -#endif - -DBGDUMP_INLINE_IF_INCLUDED void TDumpBase::String(const TStringBuf& s) { - if (s) { +#include "engine.h" + +#include <util/string/cast.h> +#include <util/string/escape.h> + +#if !defined(DBGDUMP_INLINE_IF_INCLUDED) +#define DBGDUMP_INLINE_IF_INCLUDED +#endif + +DBGDUMP_INLINE_IF_INCLUDED void TDumpBase::String(const TStringBuf& s) { + if (s) { Raw(TString(s).Quote()); - } else { - Raw("(empty)"); - } -} - -DBGDUMP_INLINE_IF_INCLUDED void TDumpBase::String(const TWtringBuf& s) { - Raw("w"); - String(ToString(s)); -} - -DBGDUMP_INLINE_IF_INCLUDED void TDumpBase::Raw(const TStringBuf& s) { + } else { + Raw("(empty)"); + } +} + +DBGDUMP_INLINE_IF_INCLUDED void TDumpBase::String(const TWtringBuf& s) { + Raw("w"); + String(ToString(s)); +} + +DBGDUMP_INLINE_IF_INCLUDED void TDumpBase::Raw(const TStringBuf& s) { Stream().Write(s.data(), s.size()); -} - -DBGDUMP_INLINE_IF_INCLUDED void TDumpBase::Char(char ch) { - Raw("'" + EscapeC(&ch, 1) + "'"); -} - -DBGDUMP_INLINE_IF_INCLUDED void TDumpBase::Char(wchar16 ch) { - Raw("w'" + ToString(EscapeC(&ch, 1)) + "'"); -} +} + +DBGDUMP_INLINE_IF_INCLUDED void TDumpBase::Char(char ch) { + Raw("'" + EscapeC(&ch, 1) + "'"); +} + +DBGDUMP_INLINE_IF_INCLUDED void TDumpBase::Char(wchar16 ch) { + Raw("w'" + ToString(EscapeC(&ch, 1)) + "'"); +} diff --git a/library/cpp/dbg_output/engine.h b/library/cpp/dbg_output/engine.h index f13c728c39..cacd9c0a8f 100644 --- a/library/cpp/dbg_output/engine.h +++ b/library/cpp/dbg_output/engine.h @@ -1,180 +1,180 @@ -#pragma once - -#include <util/stream/output.h> - +#pragma once + +#include <util/stream/output.h> + #include <utility> -#include <util/generic/strbuf.h> - -template <class T> -struct TDumper { - template <class S> - static inline void Dump(S& s, const T& t) { - s.Stream() << t; - } -}; - -namespace NDumpPrivate { - template <class T, class V> - inline void Dump(T& t, const V& v) { - ::TDumper<V>::Dump(t, v); - } - - template <class T, class V> - inline T&& operator<<(T&& t, V&& v) { - Dump(t, v); - +#include <util/generic/strbuf.h> + +template <class T> +struct TDumper { + template <class S> + static inline void Dump(S& s, const T& t) { + s.Stream() << t; + } +}; + +namespace NDumpPrivate { + template <class T, class V> + inline void Dump(T& t, const V& v) { + ::TDumper<V>::Dump(t, v); + } + + template <class T, class V> + inline T&& operator<<(T&& t, V&& v) { + Dump(t, v); + return std::forward<T>(t); - } - - struct TADLBase { - }; -} - -struct TDumpBase: public ::NDumpPrivate::TADLBase { + } + + struct TADLBase { + }; +} + +struct TDumpBase: public ::NDumpPrivate::TADLBase { inline TDumpBase(IOutputStream& out, bool indent) noexcept - : Out(&out) - , IndentLevel(0) - , Indent(indent) - { - } - + : Out(&out) + , IndentLevel(0) + , Indent(indent) + { + } + inline IOutputStream& Stream() const noexcept { - return *Out; - } - - void Char(char ch); - void Char(wchar16 ch); - - void String(const TStringBuf& s); - void String(const TWtringBuf& s); - - void Raw(const TStringBuf& s); - + return *Out; + } + + void Char(char ch); + void Char(wchar16 ch); + + void String(const TStringBuf& s); + void String(const TWtringBuf& s); + + void Raw(const TStringBuf& s); + IOutputStream* Out; - size_t IndentLevel; - bool Indent; -}; - -struct TIndentScope { - inline TIndentScope(TDumpBase& d) - : D(&d) - { - ++(D->IndentLevel); - } - + size_t IndentLevel; + bool Indent; +}; + +struct TIndentScope { + inline TIndentScope(TDumpBase& d) + : D(&d) + { + ++(D->IndentLevel); + } + inline ~TIndentScope() { - --(D->IndentLevel); - } - - TDumpBase* D; -}; - -template <class TChar> -struct TRawLiteral { + --(D->IndentLevel); + } + + TDumpBase* D; +}; + +template <class TChar> +struct TRawLiteral { const TBasicStringBuf<TChar> S; -}; - -template <class TChar> +}; + +template <class TChar> static inline TRawLiteral<TChar> DumpRaw(const TBasicStringBuf<TChar>& s) noexcept { return {s}; } template <class TChar> static inline TRawLiteral<TChar> DumpRaw(const TChar* s) noexcept { - return {s}; -} - -template <class C> -struct TDumper<TRawLiteral<C>> { - template <class S> - static inline void Dump(S& s, const TRawLiteral<C>& v) { - s.Raw(v.S); - } -}; - -struct TIndentNewLine { -}; - + return {s}; +} + +template <class C> +struct TDumper<TRawLiteral<C>> { + template <class S> + static inline void Dump(S& s, const TRawLiteral<C>& v) { + s.Raw(v.S); + } +}; + +struct TIndentNewLine { +}; + static inline TIndentNewLine IndentNewLine() noexcept { - return {}; -} - -template <> -struct TDumper<TIndentNewLine> { - template <class S> - static inline void Dump(S& s, const TIndentNewLine&) { - if (s.Indent) { + return {}; +} + +template <> +struct TDumper<TIndentNewLine> { + template <class S> + static inline void Dump(S& s, const TIndentNewLine&) { + if (s.Indent) { s << DumpRaw("\n") << DumpRaw(TString(s.IndentLevel * 4, ' ').data()); - } - } -}; - -template <class P> -struct TDumper<const P*> { - template <class S> - static inline void Dump(S& s, const P* p) { - s.Pointer(p); - } -}; - -template <class P> -struct TDumper<P*>: public TDumper<const P*> { -}; - -struct TCharDumper { - template <class S, class V> - static inline void Dump(S& s, const V& v) { - s.Char(v); - } -}; - -template <class S, class V> -static inline void OutSequence(S& s, const V& v, const char* openTag, const char* closeTag) { + } + } +}; + +template <class P> +struct TDumper<const P*> { + template <class S> + static inline void Dump(S& s, const P* p) { + s.Pointer(p); + } +}; + +template <class P> +struct TDumper<P*>: public TDumper<const P*> { +}; + +struct TCharDumper { + template <class S, class V> + static inline void Dump(S& s, const V& v) { + s.Char(v); + } +}; + +template <class S, class V> +static inline void OutSequence(S& s, const V& v, const char* openTag, const char* closeTag) { s.ColorScheme.Markup(s); - s << DumpRaw(openTag); - - { - TIndentScope scope(s); - size_t cnt = 0; - - for (const auto& x : v) { - if (cnt) { + s << DumpRaw(openTag); + + { + TIndentScope scope(s); + size_t cnt = 0; + + for (const auto& x : v) { + if (cnt) { s.ColorScheme.Markup(s); - s << DumpRaw(", "); - } - + s << DumpRaw(", "); + } + s << IndentNewLine(); s.ColorScheme.Literal(s); s << x; - ++cnt; - } - } - + ++cnt; + } + } + s << IndentNewLine(); s.ColorScheme.Markup(s); s << DumpRaw(closeTag); s.ColorScheme.ResetType(s); -} - -struct TAssocDumper { - template <class S, class V> - static inline void Dump(S& s, const V& v) { - ::OutSequence(s, v, "{", "}"); - } -}; - -struct TSeqDumper { - template <class S, class V> - static inline void Dump(S& s, const V& v) { - ::OutSequence(s, v, "[", "]"); - } -}; - -struct TStrDumper { - template <class S, class V> - static inline void Dump(S& s, const V& v) { +} + +struct TAssocDumper { + template <class S, class V> + static inline void Dump(S& s, const V& v) { + ::OutSequence(s, v, "{", "}"); + } +}; + +struct TSeqDumper { + template <class S, class V> + static inline void Dump(S& s, const V& v) { + ::OutSequence(s, v, "[", "]"); + } +}; + +struct TStrDumper { + template <class S, class V> + static inline void Dump(S& s, const V& v) { s.ColorScheme.String(s); - s.String(v); + s.String(v); s.ColorScheme.ResetType(s); - } -}; + } +}; diff --git a/library/cpp/dbg_output/ut/dbg_output_ut.cpp b/library/cpp/dbg_output/ut/dbg_output_ut.cpp index 7b285c84cb..509581e02e 100644 --- a/library/cpp/dbg_output/ut/dbg_output_ut.cpp +++ b/library/cpp/dbg_output/ut/dbg_output_ut.cpp @@ -6,24 +6,24 @@ #include <util/string/escape.h> #include <util/generic/map.h> -namespace { - struct TX { - inline TX() { - N = this; - } - - TX* N; - }; -} - -template <> -struct TDumper<TX> { - template <class S> - static inline void Dump(S& s, const TX& x) { - s << DumpRaw("x") << x.N; - } -}; - +namespace { + struct TX { + inline TX() { + N = this; + } + + TX* N; + }; +} + +template <> +struct TDumper<TX> { + template <class S> + static inline void Dump(S& s, const TX& x) { + s << DumpRaw("x") << x.N; + } +}; + namespace TMyNS { struct TMyStruct { int A, B; @@ -40,14 +40,14 @@ Y_UNIT_TEST_SUITE(TContainerPrintersTest) { Y_UNIT_TEST(TestMapCharToCharArray) { TStringStream out; - + TMap<char, const char*> m; - - m['a'] = "SMALL LETTER A"; - m['b'] = nullptr; - - out << DbgDump(m); - + + m['a'] = "SMALL LETTER A"; + m['b'] = nullptr; + + out << DbgDump(m); + UNIT_ASSERT_STRINGS_EQUAL(out.Str(), "{'a' -> \"SMALL LETTER A\", 'b' -> (empty)}"); } @@ -56,13 +56,13 @@ Y_UNIT_TEST_SUITE(TContainerPrintersTest) { TVector<TVector<wchar16>> vec(2); vec[0].push_back(0); vec[1] = {wchar16('a')}; - out << DbgDump(vec); + out << DbgDump(vec); UNIT_ASSERT_STRINGS_EQUAL(out.Str(), "[[w'\\0'], [w'a']]"); } - + Y_UNIT_TEST(TestInfinite) { - UNIT_ASSERT(!!(TStringBuilder() << DbgDumpDeep(TX()))); - } + UNIT_ASSERT(!!(TStringBuilder() << DbgDumpDeep(TX()))); + } Y_UNIT_TEST(TestLabeledDump) { TStringStream out; @@ -92,7 +92,7 @@ Y_UNIT_TEST_SUITE(TContainerPrintersTest) { "\\x1B[22;39m\\x1B[22;39m\\x1B[49m\\x1B[1;32m, \\x1B[1;31m\\x1B[42m\\x1B[1;31m\\x1B[1;33m" "\\\"b\\\"\\x1B[22;39m\\x1B[22;39m\\x1B[49m\\x1B[1;32m -> " "\\x1B[44m\\x1B[1;31m\\x1B[1;32m{\\x1B[1;31m\\x1B[1;31m2\\x1B[22;39m\\x1B[1;32m -> " - "\\x1B[1;31m'6'\\x1B[22;39m\\x1B[1;32m}\\x1B[22;39m\\x1B[22;39m\\x1B[49m\\x1B[1;32m}\\x1B[22;39m"); + "\\x1B[1;31m'6'\\x1B[22;39m\\x1B[1;32m}\\x1B[22;39m\\x1B[22;39m\\x1B[49m\\x1B[1;32m}\\x1B[22;39m"); } Y_UNIT_TEST(SmallIntOrChar) { diff --git a/library/cpp/dbg_output/ya.make b/library/cpp/dbg_output/ya.make index 7d54108f93..af1ab17c5f 100644 --- a/library/cpp/dbg_output/ya.make +++ b/library/cpp/dbg_output/ya.make @@ -1,15 +1,15 @@ -LIBRARY() - +LIBRARY() + OWNER(pg) - + PEERDIR( library/cpp/colorizer ) -SRCS( - dump.cpp - dumpers.cpp - engine.cpp -) - -END() +SRCS( + dump.cpp + dumpers.cpp + engine.cpp +) + +END() diff --git a/library/cpp/deprecated/accessors/accessors.cpp b/library/cpp/deprecated/accessors/accessors.cpp index 7d37e586fa..92f8817c3e 100644 --- a/library/cpp/deprecated/accessors/accessors.cpp +++ b/library/cpp/deprecated/accessors/accessors.cpp @@ -1 +1 @@ -#include "accessors.h" +#include "accessors.h" diff --git a/library/cpp/deprecated/accessors/accessors.h b/library/cpp/deprecated/accessors/accessors.h index 6d4b1da3ad..8ad5f56a34 100644 --- a/library/cpp/deprecated/accessors/accessors.h +++ b/library/cpp/deprecated/accessors/accessors.h @@ -3,7 +3,7 @@ #include "accessors_impl.h" namespace NAccessors { - /* + /* * Adds API compatibility between different types representing memory regions. * * i.e. this will work: @@ -23,26 +23,26 @@ namespace NAccessors { * Both will give you begin, end and size of the underlying memory region. */ - template <typename T> - inline const typename TMemoryTraits<T>::TElementType* Begin(const T& t) { - return NPrivate::TBegin<T>::Get(t); - } + template <typename T> + inline const typename TMemoryTraits<T>::TElementType* Begin(const T& t) { + return NPrivate::TBegin<T>::Get(t); + } - template <typename T> - inline const typename TMemoryTraits<T>::TElementType* End(const T& t) { - return NPrivate::TEnd<T>::Get(t); - } + template <typename T> + inline const typename TMemoryTraits<T>::TElementType* End(const T& t) { + return NPrivate::TEnd<T>::Get(t); + } - template <typename T> - inline size_t Size(const T& t) { - return End(t) - Begin(t); - } + template <typename T> + inline size_t Size(const T& t) { + return End(t) - Begin(t); + } /** * This gives some unification in terms of memory manipulation. */ - template <typename T> + template <typename T> inline void Reserve(T& t, size_t sz) { NPrivate::TReserve<T>::Do(t, sz); } @@ -75,9 +75,9 @@ namespace NAccessors { } template <typename T> - inline void Assign(T& t, - const typename TMemoryTraits<T>::TElementType* beg, - const typename TMemoryTraits<T>::TElementType* end) { - NPrivate::TAssign<T>::Do(t, beg, end); - } + inline void Assign(T& t, + const typename TMemoryTraits<T>::TElementType* beg, + const typename TMemoryTraits<T>::TElementType* end) { + NPrivate::TAssign<T>::Do(t, beg, end); + } } diff --git a/library/cpp/deprecated/accessors/accessors_impl.cpp b/library/cpp/deprecated/accessors/accessors_impl.cpp index 0bf74cab7b..f5191d647e 100644 --- a/library/cpp/deprecated/accessors/accessors_impl.cpp +++ b/library/cpp/deprecated/accessors/accessors_impl.cpp @@ -1 +1 @@ -#include "accessors_impl.h" +#include "accessors_impl.h" diff --git a/library/cpp/deprecated/accessors/accessors_impl.h b/library/cpp/deprecated/accessors/accessors_impl.h index 6b2b987351..e04cc0679f 100644 --- a/library/cpp/deprecated/accessors/accessors_impl.h +++ b/library/cpp/deprecated/accessors/accessors_impl.h @@ -3,114 +3,114 @@ #include "memory_traits.h" namespace NAccessors { - namespace NPrivate { - template <typename Ta> - struct TMemoryAccessorBase { - enum { - SimpleMemory = TMemoryTraits<Ta>::SimpleMemory, - ContinuousMemory = TMemoryTraits<Ta>::ContinuousMemory, - }; - - struct TBadAccessor; - }; - - template <typename Ta> - struct TBegin: public TMemoryAccessorBase<Ta> { + namespace NPrivate { + template <typename Ta> + struct TMemoryAccessorBase { + enum { + SimpleMemory = TMemoryTraits<Ta>::SimpleMemory, + ContinuousMemory = TMemoryTraits<Ta>::ContinuousMemory, + }; + + struct TBadAccessor; + }; + + template <typename Ta> + struct TBegin: public TMemoryAccessorBase<Ta> { using TElementType = typename TMemoryTraits<Ta>::TElementType; - template <typename Tb> - struct TNoMemoryIndirectionBegin { - static const TElementType* Get(const Tb& b) { - return (const TElementType*)&b; - } - }; + template <typename Tb> + struct TNoMemoryIndirectionBegin { + static const TElementType* Get(const Tb& b) { + return (const TElementType*)&b; + } + }; - template <typename Tb> - struct TIndirectMemoryRegionBegin { + template <typename Tb> + struct TIndirectMemoryRegionBegin { Y_HAS_MEMBER(Begin); Y_HAS_MEMBER(begin); - template <typename Tc> - struct TByBegin { - static const TElementType* Get(const Tc& b) { - return (const TElementType*)b.Begin(); - } - }; - - template <typename Tc> - struct TBybegin { - static const TElementType* Get(const Tc& b) { - return (const TElementType*)b.begin(); - } - }; - + template <typename Tc> + struct TByBegin { + static const TElementType* Get(const Tc& b) { + return (const TElementType*)b.Begin(); + } + }; + + template <typename Tc> + struct TBybegin { + static const TElementType* Get(const Tc& b) { + return (const TElementType*)b.begin(); + } + }; + using TGet = std::conditional_t<THasBegin<Tb>::value, TByBegin<Tb>, TBybegin<Tb>>; - - static const TElementType* Get(const Tb& b) { - return TGet::Get(b); - } - }; - + + static const TElementType* Get(const Tb& b) { + return TGet::Get(b); + } + }; + using TGet = std::conditional_t< - TMemoryAccessorBase<Ta>::SimpleMemory, - TNoMemoryIndirectionBegin<Ta>, + TMemoryAccessorBase<Ta>::SimpleMemory, + TNoMemoryIndirectionBegin<Ta>, std::conditional_t< - TMemoryAccessorBase<Ta>::ContinuousMemory, - TIndirectMemoryRegionBegin<Ta>, + TMemoryAccessorBase<Ta>::ContinuousMemory, + TIndirectMemoryRegionBegin<Ta>, typename TMemoryAccessorBase<Ta>::TBadAccessor>>; - - static const TElementType* Get(const Ta& b) { - return TGet::Get(b); + + static const TElementType* Get(const Ta& b) { + return TGet::Get(b); } }; - template <typename Ta> - struct TEnd: public TMemoryAccessorBase<Ta> { + template <typename Ta> + struct TEnd: public TMemoryAccessorBase<Ta> { using TElementType = typename TMemoryTraits<Ta>::TElementType; - template <typename Tb> - struct TNoMemoryIndirectionEnd { - static const TElementType* Get(const Tb& b) { - return (const TElementType*)(&b + 1); - } - }; + template <typename Tb> + struct TNoMemoryIndirectionEnd { + static const TElementType* Get(const Tb& b) { + return (const TElementType*)(&b + 1); + } + }; - template <typename Tb> - struct TIndirectMemoryRegionEnd { + template <typename Tb> + struct TIndirectMemoryRegionEnd { Y_HAS_MEMBER(End); Y_HAS_MEMBER(end); - template <typename Tc> - struct TByEnd { - static const TElementType* Get(const Tc& b) { - return (const TElementType*)b.End(); - } - }; + template <typename Tc> + struct TByEnd { + static const TElementType* Get(const Tc& b) { + return (const TElementType*)b.End(); + } + }; - template <typename Tc> - struct TByend { - static const TElementType* Get(const Tc& b) { - return (const TElementType*)b.end(); - } - }; + template <typename Tc> + struct TByend { + static const TElementType* Get(const Tc& b) { + return (const TElementType*)b.end(); + } + }; using TGet = std::conditional_t<THasEnd<Tb>::value, TByEnd<Tb>, TByend<Tb>>; - static const TElementType* Get(const Tb& b) { - return TGet::Get(b); - } - }; + static const TElementType* Get(const Tb& b) { + return TGet::Get(b); + } + }; using TGet = std::conditional_t< - TMemoryAccessorBase<Ta>::SimpleMemory, - TNoMemoryIndirectionEnd<Ta>, + TMemoryAccessorBase<Ta>::SimpleMemory, + TNoMemoryIndirectionEnd<Ta>, std::conditional_t< - TMemoryAccessorBase<Ta>::ContinuousMemory, - TIndirectMemoryRegionEnd<Ta>, + TMemoryAccessorBase<Ta>::ContinuousMemory, + TIndirectMemoryRegionEnd<Ta>, typename TMemoryAccessorBase<Ta>::TBadAccessor>>; - static const TElementType* Get(const Ta& b) { - return TGet::Get(b); + static const TElementType* Get(const Ta& b) { + return TGet::Get(b); } }; @@ -343,13 +343,13 @@ namespace NAccessors { } }; - template <typename Ta> - struct TAssign: public TMemoryAccessorBase<Ta> { + template <typename Ta> + struct TAssign: public TMemoryAccessorBase<Ta> { using TElementType = typename TMemoryTraits<Ta>::TElementType; - template <typename Tb> - struct TNoMemoryIndirectionAssign { - static void Do(Tb& b, const TElementType* beg, const TElementType* end) { + template <typename Tb> + struct TNoMemoryIndirectionAssign { + static void Do(Tb& b, const TElementType* beg, const TElementType* end) { if (sizeof(Tb) == sizeof(TElementType) && end - beg > 0) { memcpy(&b, beg, sizeof(Tb)); } else if (end - beg > 0) { @@ -357,63 +357,63 @@ namespace NAccessors { } else { Zero(b); } - } - }; + } + }; - template <typename Tb> - struct TIndirectMemoryRegionAssign { + template <typename Tb> + struct TIndirectMemoryRegionAssign { Y_HAS_MEMBER(Assign); Y_HAS_MEMBER(assign); - template <typename Tc> - struct TByAssign { - static void Do(Tc& b, const TElementType* beg, const TElementType* end) { - b.Assign(beg, end); - } - }; - - template <typename Tc> - struct TByassign { - static void Do(Tc& b, const TElementType* beg, const TElementType* end) { - b.assign(beg, end); - } - }; - - template <typename Tc> - struct TByClearAppend { - static void Do(Tc& b, const TElementType* beg, const TElementType* end) { - TClear<Tc, false>::Do(b); - TAppendRegion<Tc>::Do(b, beg, end); - } - }; - - template <typename Tc> - struct TByConstruction { - static void Do(Tc& b, const TElementType* beg, const TElementType* end) { - b = Tc(beg, end); - } - }; + template <typename Tc> + struct TByAssign { + static void Do(Tc& b, const TElementType* beg, const TElementType* end) { + b.Assign(beg, end); + } + }; + + template <typename Tc> + struct TByassign { + static void Do(Tc& b, const TElementType* beg, const TElementType* end) { + b.assign(beg, end); + } + }; + + template <typename Tc> + struct TByClearAppend { + static void Do(Tc& b, const TElementType* beg, const TElementType* end) { + TClear<Tc, false>::Do(b); + TAppendRegion<Tc>::Do(b, beg, end); + } + }; + + template <typename Tc> + struct TByConstruction { + static void Do(Tc& b, const TElementType* beg, const TElementType* end) { + b = Tc(beg, end); + } + }; using TDo = std::conditional_t< THasAssign<Tb>::value, - TByAssign<Tb>, + TByAssign<Tb>, std::conditional_t< THasassign<Tb>::value, - TByassign<Tb>, + TByassign<Tb>, std::conditional_t< - TMemoryTraits<Tb>::OwnsMemory, - TByClearAppend<Tb>, + TMemoryTraits<Tb>::OwnsMemory, + TByClearAppend<Tb>, TByConstruction<Tb>>>>; - static void Do(Tb& b, const TElementType* beg, const TElementType* end) { - TDo::Do(b, beg, end); - } - }; + static void Do(Tb& b, const TElementType* beg, const TElementType* end) { + TDo::Do(b, beg, end); + } + }; using TDo = std::conditional_t<TMemoryAccessorBase<Ta>::SimpleMemory, TNoMemoryIndirectionAssign<Ta>, TIndirectMemoryRegionAssign<Ta>>; - static void Do(Ta& b, const TElementType* beg, const TElementType* end) { - TDo::Do(b, beg, end); + static void Do(Ta& b, const TElementType* beg, const TElementType* end) { + TDo::Do(b, beg, end); } }; } diff --git a/library/cpp/deprecated/accessors/accessors_ut.cpp b/library/cpp/deprecated/accessors/accessors_ut.cpp index a9bdc9fcc4..9d993b10bf 100644 --- a/library/cpp/deprecated/accessors/accessors_ut.cpp +++ b/library/cpp/deprecated/accessors/accessors_ut.cpp @@ -1,17 +1,17 @@ #include "accessors.h" #include <library/cpp/testing/unittest/registar.h> - + #include <util/generic/buffer.h> #include <util/generic/vector.h> #include <array> class TAccessorsTest: public TTestBase { - UNIT_TEST_SUITE(TAccessorsTest); - UNIT_TEST(TestAccessors); - UNIT_TEST_SUITE_END(); - + UNIT_TEST_SUITE(TAccessorsTest); + UNIT_TEST(TestAccessors); + UNIT_TEST_SUITE_END(); + private: template <typename T> void TestRead(const T& t, const char* comm) { @@ -50,7 +50,7 @@ private: TestRead('a', "char"); TestRead(1, "int"); - int t[4] = {0, 1, 2, 3}; + int t[4] = {0, 1, 2, 3}; TestRead(t, "int[4]"); diff --git a/library/cpp/deprecated/accessors/memory_traits.cpp b/library/cpp/deprecated/accessors/memory_traits.cpp index df53026cf4..76d0d3bad4 100644 --- a/library/cpp/deprecated/accessors/memory_traits.cpp +++ b/library/cpp/deprecated/accessors/memory_traits.cpp @@ -1 +1 @@ -#include "memory_traits.h" +#include "memory_traits.h" diff --git a/library/cpp/deprecated/accessors/memory_traits.h b/library/cpp/deprecated/accessors/memory_traits.h index aa837705d3..1ebf53f108 100644 --- a/library/cpp/deprecated/accessors/memory_traits.h +++ b/library/cpp/deprecated/accessors/memory_traits.h @@ -1,13 +1,13 @@ #pragma once #include <util/generic/array_ref.h> -#include <util/memory/blob.h> -#include <util/memory/tempbuf.h> +#include <util/memory/blob.h> +#include <util/memory/tempbuf.h> #include <util/generic/buffer.h> #include <util/generic/strbuf.h> #include <util/generic/string.h> -#include <util/generic/vector.h> -#include <util/generic/typetraits.h> +#include <util/generic/vector.h> +#include <util/generic/typetraits.h> #include <array> #include <string> @@ -47,14 +47,14 @@ struct TMemoryTraits<std::array<T, n>> { }; template <typename A, typename B> -struct TMemoryTraits<std::pair<A, B>> { +struct TMemoryTraits<std::pair<A, B>> { enum { SimpleMemory = TMemoryTraits<A>::SimpleMemory && TMemoryTraits<B>::SimpleMemory, ContinuousMemory = SimpleMemory, OwnsMemory = SimpleMemory, }; - using TElementType = std::pair<A, B>; + using TElementType = std::pair<A, B>; }; template <> @@ -80,7 +80,7 @@ struct TMemoryTraits<TTempBuf> { }; template <> -struct TMemoryTraits< ::TBlob> { +struct TMemoryTraits< ::TBlob> { enum { SimpleMemory = false, ContinuousMemory = true, @@ -101,7 +101,7 @@ struct TElementDependentMemoryTraits { }; template <typename T, typename TAlloc> -struct TMemoryTraits<std::vector<T, TAlloc>>: public TElementDependentMemoryTraits<T> { +struct TMemoryTraits<std::vector<T, TAlloc>>: public TElementDependentMemoryTraits<T> { enum { OwnsMemory = TMemoryTraits<T>::OwnsMemory }; @@ -112,14 +112,14 @@ struct TMemoryTraits<TVector<T, TAlloc>>: public TMemoryTraits<std::vector<T, TA }; template <typename T> -struct TMemoryTraits<TTempArray<T>>: public TElementDependentMemoryTraits<T> { +struct TMemoryTraits<TTempArray<T>>: public TElementDependentMemoryTraits<T> { enum { OwnsMemory = TMemoryTraits<T>::OwnsMemory }; }; template <typename T, typename TCharTraits, typename TAlloc> -struct TMemoryTraits<std::basic_string<T, TCharTraits, TAlloc>>: public TElementDependentMemoryTraits<T> { +struct TMemoryTraits<std::basic_string<T, TCharTraits, TAlloc>>: public TElementDependentMemoryTraits<T> { enum { OwnsMemory = TMemoryTraits<T>::OwnsMemory }; @@ -154,7 +154,7 @@ struct TMemoryTraits<TBasicStringBuf<TCharType, TCharTraits>>: public TElementDe }; template <> -struct TMemoryTraits<TStringBuf>: public TElementDependentMemoryTraits<char> { +struct TMemoryTraits<TStringBuf>: public TElementDependentMemoryTraits<char> { enum { OwnsMemory = false }; diff --git a/library/cpp/deprecated/enum_codegen/enum_codegen.cpp b/library/cpp/deprecated/enum_codegen/enum_codegen.cpp index 3931b05924..21ce9e6334 100644 --- a/library/cpp/deprecated/enum_codegen/enum_codegen.cpp +++ b/library/cpp/deprecated/enum_codegen/enum_codegen.cpp @@ -1 +1 @@ -#include "enum_codegen.h" +#include "enum_codegen.h" diff --git a/library/cpp/deprecated/enum_codegen/enum_codegen.h b/library/cpp/deprecated/enum_codegen/enum_codegen.h index dfb04ecac2..2f050d91bd 100644 --- a/library/cpp/deprecated/enum_codegen/enum_codegen.h +++ b/library/cpp/deprecated/enum_codegen/enum_codegen.h @@ -5,29 +5,29 @@ #define ENUM_VALUE_GEN(name, value, ...) name = value, #define ENUM_VALUE_GEN_NO_VALUE(name, ...) name, -#define ENUM_TO_STRING_IMPL_ITEM(name, ...) \ - case name: \ - return #name; -#define ENUM_LTLT_IMPL_ITEM(name, ...) \ - case name: \ - os << #name; \ - break; +#define ENUM_TO_STRING_IMPL_ITEM(name, ...) \ + case name: \ + return #name; +#define ENUM_LTLT_IMPL_ITEM(name, ...) \ + case name: \ + os << #name; \ + break; -#define ENUM_TO_STRING(type, MAP) \ - static inline const char* ToCString(type value) { \ - switch (value) { \ - MAP(ENUM_TO_STRING_IMPL_ITEM) \ - default: \ - return "UNKNOWN"; \ - } \ - } \ - \ +#define ENUM_TO_STRING(type, MAP) \ + static inline const char* ToCString(type value) { \ + switch (value) { \ + MAP(ENUM_TO_STRING_IMPL_ITEM) \ + default: \ + return "UNKNOWN"; \ + } \ + } \ + \ static inline IOutputStream& operator<<(IOutputStream& os, type value) { \ - switch (value) { \ - MAP(ENUM_LTLT_IMPL_ITEM) \ - default: \ - os << int(value); \ - break; \ - } \ - return os; \ - } + switch (value) { \ + MAP(ENUM_LTLT_IMPL_ITEM) \ + default: \ + os << int(value); \ + break; \ + } \ + return os; \ + } diff --git a/library/cpp/deprecated/enum_codegen/enum_codegen_ut.cpp b/library/cpp/deprecated/enum_codegen/enum_codegen_ut.cpp index f8f1c9b6df..10f9c597e6 100644 --- a/library/cpp/deprecated/enum_codegen/enum_codegen_ut.cpp +++ b/library/cpp/deprecated/enum_codegen/enum_codegen_ut.cpp @@ -2,11 +2,11 @@ #include "enum_codegen.h" -#include <util/string/builder.h> - +#include <util/string/builder.h> + #define COLOR_MAP(XX) \ - XX(RED) \ - XX(GREEN) \ + XX(RED) \ + XX(GREEN) \ XX(BLUE) enum EColor { @@ -16,8 +16,8 @@ enum EColor { ENUM_TO_STRING(EColor, COLOR_MAP) #define MULTIPLIER_MAP(XX) \ - XX(GB, 9) \ - XX(MB, 6) \ + XX(GB, 9) \ + XX(MB, 6) \ XX(KB, 3) enum EMultiplier { @@ -34,7 +34,7 @@ Y_UNIT_TEST_SUITE(EnumCodegen) { Y_UNIT_TEST(ToCString) { UNIT_ASSERT_VALUES_EQUAL("RED", ToCString(RED)); UNIT_ASSERT_VALUES_EQUAL("BLUE", ToCString(BLUE)); - UNIT_ASSERT_VALUES_EQUAL("GREEN", (TStringBuilder() << GREEN)); - UNIT_ASSERT_VALUES_EQUAL("GB", ToCString(GB)); + UNIT_ASSERT_VALUES_EQUAL("GREEN", (TStringBuilder() << GREEN)); + UNIT_ASSERT_VALUES_EQUAL("GB", ToCString(GB)); } } diff --git a/library/cpp/deprecated/kmp/kmp.cpp b/library/cpp/deprecated/kmp/kmp.cpp index d02074c94a..cce93acc4b 100644 --- a/library/cpp/deprecated/kmp/kmp.cpp +++ b/library/cpp/deprecated/kmp/kmp.cpp @@ -1,5 +1,5 @@ -#include "kmp.h" - +#include "kmp.h" + #include <util/generic/yexception.h> TKMPMatcher::TKMPMatcher(const char* patternBegin, const char* patternEnd) diff --git a/library/cpp/deprecated/kmp/kmp.h b/library/cpp/deprecated/kmp/kmp.h index a7f72eece6..4689697738 100644 --- a/library/cpp/deprecated/kmp/kmp.h +++ b/library/cpp/deprecated/kmp/kmp.h @@ -3,9 +3,9 @@ #include <util/generic/ptr.h> #include <util/generic/string.h> #include <util/generic/vector.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> -template <typename T> +template <typename T> void ComputePrefixFunction(const T* begin, const T* end, ssize_t** result) { Y_ENSURE(begin != end, TStringBuf("empty pattern")); ssize_t len = end - begin; @@ -58,7 +58,7 @@ public: } }; -template <typename T> +template <typename T> class TKMPStreamMatcher { public: class ICallback { diff --git a/library/cpp/deprecated/kmp/kmp_ut.cpp b/library/cpp/deprecated/kmp/kmp_ut.cpp index c2eda83c57..8969cc1642 100644 --- a/library/cpp/deprecated/kmp/kmp_ut.cpp +++ b/library/cpp/deprecated/kmp/kmp_ut.cpp @@ -1,8 +1,8 @@ -#include "kmp.h" - +#include "kmp.h" + #include <library/cpp/testing/unittest/registar.h> -#include <util/stream/output.h> +#include <util/stream/output.h> static TVector<int> FindAll(const TString& pattern, const TString& string) { TVector<int> result; @@ -17,12 +17,12 @@ static TVector<int> FindAll(const TString& pattern, const TString& string) { return result; } -class TTestKMP: public TTestBase { +class TTestKMP: public TTestBase { UNIT_TEST_SUITE(TTestKMP); - UNIT_TEST(Test); - UNIT_TEST(TestStream); + UNIT_TEST(Test); + UNIT_TEST(TestStream); UNIT_TEST_SUITE_END(); - + public: void Test() { TVector<int> ans = {0, 2}; @@ -35,12 +35,12 @@ public: UNIT_ASSERT_EQUAL(FindAll("ab", "abab"), ans); } - class TKMPSimpleCallback: public TKMPStreamMatcher<int>::ICallback { + class TKMPSimpleCallback: public TKMPStreamMatcher<int>::ICallback { private: int* Begin; int* End; int Count; - + public: TKMPSimpleCallback(int* begin, int* end) : Begin(begin) diff --git a/library/cpp/deprecated/mapped_file/mapped_file.cpp b/library/cpp/deprecated/mapped_file/mapped_file.cpp index b0e4511299..a11b79783d 100644 --- a/library/cpp/deprecated/mapped_file/mapped_file.cpp +++ b/library/cpp/deprecated/mapped_file/mapped_file.cpp @@ -5,9 +5,9 @@ #include <util/system/hi_lo.h> #include <util/system/filemap.h> -TMappedFile::TMappedFile(TFileMap* map, const char* dbgName) { - Map_ = map; - i64 len = Map_->Length(); +TMappedFile::TMappedFile(TFileMap* map, const char* dbgName) { + Map_ = map; + i64 len = Map_->Length(); if (Hi32(len) != 0 && sizeof(size_t) <= sizeof(ui32)) ythrow yexception() << "File '" << dbgName << "' mapping error: " << len << " too large"; @@ -36,7 +36,7 @@ void TMappedFile::init(const TString& name) { } void TMappedFile::init(const TString& name, size_t length, TFileMap::EOpenMode om) { - THolder<TFileMap> map(new TFileMap(name, length, om)); + THolder<TFileMap> map(new TFileMap(name, length, om)); TMappedFile newFile(map.Get(), name.data()); Y_UNUSED(map.Release()); newFile.swap(*this); @@ -44,7 +44,7 @@ void TMappedFile::init(const TString& name, size_t length, TFileMap::EOpenMode o } void TMappedFile::init(const TFile& file, TFileMap::EOpenMode om, const char* dbgName) { - THolder<TFileMap> map(new TFileMap(file, om)); + THolder<TFileMap> map(new TFileMap(file, om)); TMappedFile newFile(map.Get(), dbgName); Y_UNUSED(map.Release()); newFile.swap(*this); @@ -52,13 +52,13 @@ void TMappedFile::init(const TFile& file, TFileMap::EOpenMode om, const char* db } void TMappedFile::init(const TString& name, TFileMap::EOpenMode om) { - THolder<TFileMap> map(new TFileMap(name, om)); + THolder<TFileMap> map(new TFileMap(name, om)); TMappedFile newFile(map.Get(), name.data()); Y_UNUSED(map.Release()); newFile.swap(*this); newFile.term(); } -void TMappedFile::flush() { - Map_->Flush(); +void TMappedFile::flush() { + Map_->Flush(); } diff --git a/library/cpp/deprecated/mapped_file/mapped_file.h b/library/cpp/deprecated/mapped_file/mapped_file.h index 45859ed65a..4b9c0c8c52 100644 --- a/library/cpp/deprecated/mapped_file/mapped_file.h +++ b/library/cpp/deprecated/mapped_file/mapped_file.h @@ -9,24 +9,24 @@ #include <util/system/file.h> #include <util/system/filemap.h> #include <util/system/yassert.h> - + #include <cstdio> -#include <new> - +#include <new> + /// Deprecated (by pg@), use TFileMap or TMemoryMap instead -class TMappedFile { +class TMappedFile { private: - TFileMap* Map_; + TFileMap* Map_; private: - TMappedFile(TFileMap* map, const char* dbgName); + TMappedFile(TFileMap* map, const char* dbgName); public: - TMappedFile() { + TMappedFile() { Map_ = nullptr; } - ~TMappedFile() { + ~TMappedFile() { term(); } @@ -47,16 +47,16 @@ public: void flush(); - void term() { - if (Map_) { - Map_->Unmap(); - delete Map_; + void term() { + if (Map_) { + Map_->Unmap(); + delete Map_; Map_ = nullptr; } } - size_t getSize() const { - return (Map_ ? Map_->MappedSize() : 0); + size_t getSize() const { + return (Map_ ? Map_->MappedSize() : 0); } void* getData(size_t pos = 0) const { @@ -67,6 +67,6 @@ public: void precharge(size_t pos = 0, size_t size = (size_t)-1) const; void swap(TMappedFile& file) noexcept { - DoSwap(Map_, file.Map_); + DoSwap(Map_, file.Map_); } }; diff --git a/library/cpp/deprecated/mapped_file/ut/mapped_file_ut.cpp b/library/cpp/deprecated/mapped_file/ut/mapped_file_ut.cpp index afbd5b3358..1f40de11a3 100644 --- a/library/cpp/deprecated/mapped_file/ut/mapped_file_ut.cpp +++ b/library/cpp/deprecated/mapped_file/ut/mapped_file_ut.cpp @@ -1,6 +1,6 @@ #include <library/cpp/deprecated/mapped_file/mapped_file.h> #include <library/cpp/testing/unittest/registar.h> - + #include <util/system/fs.h> Y_UNIT_TEST_SUITE(TMappedFileTest) { diff --git a/library/cpp/deprecated/split/delim_string_iter.h b/library/cpp/deprecated/split/delim_string_iter.h index 8e4ca171a0..9ef4bada93 100644 --- a/library/cpp/deprecated/split/delim_string_iter.h +++ b/library/cpp/deprecated/split/delim_string_iter.h @@ -3,10 +3,10 @@ #include <util/generic/algorithm.h> #include <util/generic/strbuf.h> #include <util/generic/yexception.h> -#include <util/string/cast.h> -#include <util/system/yassert.h> +#include <util/string/cast.h> +#include <util/system/yassert.h> -#include <iterator> +#include <iterator> class TDelimStringIter { public: @@ -18,8 +18,8 @@ public: inline TDelimStringIter(const char* begin, const char* strEnd, TStringBuf delim) : TDelimStringIter(TStringBuf(begin, strEnd), delim) - { - } + { + } inline TDelimStringIter(TStringBuf str, TStringBuf delim) : IsValid(true) @@ -31,10 +31,10 @@ public: inline TDelimStringIter() : IsValid(false) - { - } + { + } - inline explicit operator bool() const { + inline explicit operator bool() const { return IsValid; } @@ -52,7 +52,7 @@ public: } inline void operator+=(size_t n) { - for (; n > 0; --n) { + for (; n > 0; --n) { ++(*this); } } @@ -74,7 +74,7 @@ public: } // Get & advance - template <class T> + template <class T> inline bool TryNext(T& t) { if (IsValid) { t = FromString<T>(Current); @@ -85,7 +85,7 @@ public: } } - template <class T> + template <class T> inline TDelimStringIter& Next(T& t) // Get & advance { if (!TryNext(t)) @@ -93,7 +93,7 @@ public: return *this; } - template <class T> + template <class T> inline T GetNext() { T res; Next(res); @@ -124,7 +124,7 @@ public: private: inline void UpdateCurrent() { // it is much faster than TStringBuf::find - size_t pos = std::search(Str.begin(), Str.end(), Delim.begin(), Delim.end()) - Str.begin(); + size_t pos = std::search(Str.begin(), Str.end(), Delim.begin(), Delim.end()) - Str.begin(); Current = Str.Head(pos); } @@ -137,15 +137,15 @@ private: }; //example: for (TStringBuf field: TDelimStroka(line, "@@")) { ... } -struct TDelimStroka { +struct TDelimStroka { TStringBuf S; TStringBuf Delim; inline TDelimStroka(TStringBuf s, TStringBuf delim) : S(s) , Delim(delim) - { - } + { + } inline TDelimStringIter begin() const { return TDelimStringIter(S, Delim); diff --git a/library/cpp/deprecated/split/delim_string_iter_ut.cpp b/library/cpp/deprecated/split/delim_string_iter_ut.cpp index 18a8b2a160..72ba1c4f69 100644 --- a/library/cpp/deprecated/split/delim_string_iter_ut.cpp +++ b/library/cpp/deprecated/split/delim_string_iter_ut.cpp @@ -43,7 +43,7 @@ Y_UNIT_TEST_SUITE(TDelimStrokaIterTestSuite) { TVector<TStringBuf> expected = {"1", "", "3@4", ""}; TVector<TStringBuf> got; - for (TStringBuf x : TDelimStroka("1@@@@3@4@@", "@@")) { + for (TStringBuf x : TDelimStroka("1@@@@3@4@@", "@@")) { got.push_back(x); } @@ -70,30 +70,30 @@ Y_UNIT_TEST_SUITE(TKeyValueDelimStringIterTestSuite) { Y_UNIT_TEST(SingleCharacterAsDelimiter) { AssertKeyValueStringSplit( "abc=123,cde=qwer", ",", - {{"abc", "123"}, - {"cde", "qwer"}}); + {{"abc", "123"}, + {"cde", "qwer"}}); } Y_UNIT_TEST(MultipleCharactersAsDelimiter) { AssertKeyValueStringSplit( "abc=xyz@@qwerty=zxcv", "@@", - {{"abc", "xyz"}, - {"qwerty", "zxcv"}}); + {{"abc", "xyz"}, + {"qwerty", "zxcv"}}); } Y_UNIT_TEST(NoDelimiters) { AssertKeyValueStringSplit( "abc=zz", ",", - {{"abc", "zz"}}); + {{"abc", "zz"}}); } Y_UNIT_TEST(EmptyElements) { AssertKeyValueStringSplit( "@@abc=zxy@@@@qwerty=y@@", "@@", - {{"", ""}, - {"abc", "zxy"}, - {"", ""}, - {"qwerty", "y"}, - {"", ""}}); + {{"", ""}, + {"abc", "zxy"}, + {"", ""}, + {"qwerty", "y"}, + {"", ""}}); } } diff --git a/library/cpp/deprecated/split/split_iterator.cpp b/library/cpp/deprecated/split/split_iterator.cpp index 32262d25bd..b1fe9a5f15 100644 --- a/library/cpp/deprecated/split/split_iterator.cpp +++ b/library/cpp/deprecated/split/split_iterator.cpp @@ -1,23 +1,23 @@ -#include "split_iterator.h" - -#include <util/system/yassert.h> - -#include <cctype> -#include <cstring> -#include <cstdlib> - +#include "split_iterator.h" + +#include <util/system/yassert.h> + +#include <cctype> +#include <cstring> +#include <cstdlib> + /****************** TSplitDelimiters2 ******************/ -TSplitDelimiters::TSplitDelimiters(const char* s) { +TSplitDelimiters::TSplitDelimiters(const char* s) { memset(Delims, 0, sizeof(Delims)); while (*s) - Delims[(ui8) * (s++)] = true; + Delims[(ui8) * (s++)] = true; } /****************** TSplitBase ******************/ -TSplitBase::TSplitBase(const char* str, size_t length) - : Str(str) - , Len(length) +TSplitBase::TSplitBase(const char* str, size_t length) + : Str(str) + , Len(length) { } @@ -29,62 +29,62 @@ TSplitBase::TSplitBase(const TString& s) /****************** TDelimitersSplit ******************/ -TDelimitersSplit::TDelimitersSplit(const char* str, size_t length, const TSplitDelimiters& delimiters) - : TSplitBase(str, length) - , Delimiters(delimiters) +TDelimitersSplit::TDelimitersSplit(const char* str, size_t length, const TSplitDelimiters& delimiters) + : TSplitBase(str, length) + , Delimiters(delimiters) { } TDelimitersSplit::TDelimitersSplit(const TString& s, const TSplitDelimiters& delimiters) - : TSplitBase(s) - , Delimiters(delimiters) + : TSplitBase(s) + , Delimiters(delimiters) { } -size_t TDelimitersSplit::Begin() const { +size_t TDelimitersSplit::Begin() const { size_t pos = 0; - while ((pos < Len) && Delimiters.IsDelimiter(Str[pos])) + while ((pos < Len) && Delimiters.IsDelimiter(Str[pos])) ++pos; return pos; } -TSizeTRegion TDelimitersSplit::Next(size_t& pos) const { +TSizeTRegion TDelimitersSplit::Next(size_t& pos) const { size_t begin = pos; - while ((pos < Len) && !Delimiters.IsDelimiter(Str[pos])) + while ((pos < Len) && !Delimiters.IsDelimiter(Str[pos])) ++pos; TSizeTRegion result(begin, pos); - while ((pos < Len) && Delimiters.IsDelimiter(Str[pos])) + while ((pos < Len) && Delimiters.IsDelimiter(Str[pos])) ++pos; - + return result; } -TDelimitersSplit::TIterator TDelimitersSplit::Iterator() const { +TDelimitersSplit::TIterator TDelimitersSplit::Iterator() const { return TIterator(*this); } /****************** TDelimitersStrictSplit ******************/ -TDelimitersStrictSplit::TDelimitersStrictSplit(const char* str, size_t length, const TSplitDelimiters& delimiters) - : TSplitBase(str, length) - , Delimiters(delimiters) +TDelimitersStrictSplit::TDelimitersStrictSplit(const char* str, size_t length, const TSplitDelimiters& delimiters) + : TSplitBase(str, length) + , Delimiters(delimiters) { } TDelimitersStrictSplit::TDelimitersStrictSplit(const TString& s, const TSplitDelimiters& delimiters) - : TSplitBase(s) - , Delimiters(delimiters) + : TSplitBase(s) + , Delimiters(delimiters) { } -TDelimitersStrictSplit::TIterator TDelimitersStrictSplit::Iterator() const { +TDelimitersStrictSplit::TIterator TDelimitersStrictSplit::Iterator() const { return TIterator(*this); } -TSizeTRegion TDelimitersStrictSplit::Next(size_t& pos) const { +TSizeTRegion TDelimitersStrictSplit::Next(size_t& pos) const { size_t begin = pos; - while ((pos < Len) && !Delimiters.IsDelimiter(Str[pos])) + while ((pos < Len) && !Delimiters.IsDelimiter(Str[pos])) ++pos; TSizeTRegion result(begin, pos); @@ -94,7 +94,7 @@ TSizeTRegion TDelimitersStrictSplit::Next(size_t& pos) const { return result; } -size_t TDelimitersStrictSplit::Begin() const { +size_t TDelimitersStrictSplit::Begin() const { return 0; } @@ -142,84 +142,84 @@ size_t TScreenedDelimitersSplit::Begin() const { /****************** TDelimitersSplitWithoutTags ******************/ -TDelimitersSplitWithoutTags::TDelimitersSplitWithoutTags(const char* str, size_t length, const TSplitDelimiters& delimiters) - : TSplitBase(str, length) - , Delimiters(delimiters) +TDelimitersSplitWithoutTags::TDelimitersSplitWithoutTags(const char* str, size_t length, const TSplitDelimiters& delimiters) + : TSplitBase(str, length) + , Delimiters(delimiters) { } TDelimitersSplitWithoutTags::TDelimitersSplitWithoutTags(const TString& s, const TSplitDelimiters& delimiters) - : TSplitBase(s) - , Delimiters(delimiters) + : TSplitBase(s) + , Delimiters(delimiters) { } -size_t TDelimitersSplitWithoutTags::SkipTag(size_t pos) const { +size_t TDelimitersSplitWithoutTags::SkipTag(size_t pos) const { Y_ASSERT('<' == Str[pos]); while ((pos < Len) && ('>' != Str[pos])) ++pos; return pos + 1; } -size_t TDelimitersSplitWithoutTags::SkipDelimiters(size_t pos) const { - while (true) { - while ((pos < Len) && Delimiters.IsDelimiter(Str[pos]) && ('<' != Str[pos])) +size_t TDelimitersSplitWithoutTags::SkipDelimiters(size_t pos) const { + while (true) { + while ((pos < Len) && Delimiters.IsDelimiter(Str[pos]) && ('<' != Str[pos])) ++pos; - if (pos < Len) { + if (pos < Len) { if ('<' != Str[pos]) break; else pos = SkipTag(pos); - } else + } else break; } return pos; } -size_t TDelimitersSplitWithoutTags::Begin() const { +size_t TDelimitersSplitWithoutTags::Begin() const { size_t pos = 0; pos = SkipDelimiters(pos); return pos; } -TSizeTRegion TDelimitersSplitWithoutTags::Next(size_t& pos) const { +TSizeTRegion TDelimitersSplitWithoutTags::Next(size_t& pos) const { size_t begin = pos; - while ((pos < Len) && !Delimiters.IsDelimiter(Str[pos]) && ('<' != Str[pos])) + while ((pos < Len) && !Delimiters.IsDelimiter(Str[pos]) && ('<' != Str[pos])) ++pos; TSizeTRegion result(begin, pos); - + pos = SkipDelimiters(pos); return result; } -TDelimitersSplitWithoutTags::TIterator TDelimitersSplitWithoutTags::Iterator() const { +TDelimitersSplitWithoutTags::TIterator TDelimitersSplitWithoutTags::Iterator() const { return TIterator(*this); } /****************** TCharSplit ******************/ -TCharSplit::TCharSplit(const char* str, size_t length) - : TSplitBase(str, length) +TCharSplit::TCharSplit(const char* str, size_t length) + : TSplitBase(str, length) { } TCharSplit::TCharSplit(const TString& s) - : TSplitBase(s) + : TSplitBase(s) { } -TCharSplit::TIterator TCharSplit::Iterator() const { +TCharSplit::TIterator TCharSplit::Iterator() const { return TIterator(*this); } -TSizeTRegion TCharSplit::Next(size_t& pos) const { +TSizeTRegion TCharSplit::Next(size_t& pos) const { TSizeTRegion result(pos, pos + 1); ++pos; return result; } -size_t TCharSplit::Begin() const { +size_t TCharSplit::Begin() const { return 0; } @@ -235,33 +235,33 @@ TCharSplitWithoutTags::TCharSplitWithoutTags(const TString& s) { } -size_t TCharSplitWithoutTags::SkipTag(size_t pos) const { +size_t TCharSplitWithoutTags::SkipTag(size_t pos) const { Y_ASSERT('<' == Str[pos]); while ((pos < Len) && ('>' != Str[pos])) ++pos; return pos + 1; } -size_t TCharSplitWithoutTags::SkipDelimiters(size_t pos) const { - while (true) { - if (pos < Len) { +size_t TCharSplitWithoutTags::SkipDelimiters(size_t pos) const { + while (true) { + if (pos < Len) { if ('<' != Str[pos]) break; else pos = SkipTag(pos); - } else + } else break; } return pos; } -size_t TCharSplitWithoutTags::Begin() const { +size_t TCharSplitWithoutTags::Begin() const { size_t pos = 0; pos = SkipDelimiters(pos); return pos; } -TSizeTRegion TCharSplitWithoutTags::Next(size_t& pos) const { +TSizeTRegion TCharSplitWithoutTags::Next(size_t& pos) const { size_t begin = pos++; TSizeTRegion result(begin, pos); @@ -270,7 +270,7 @@ TSizeTRegion TCharSplitWithoutTags::Next(size_t& pos) const { return result; } -TCharSplitWithoutTags::TIterator TCharSplitWithoutTags::Iterator() const { +TCharSplitWithoutTags::TIterator TCharSplitWithoutTags::Iterator() const { return TIterator(*this); } diff --git a/library/cpp/deprecated/split/split_iterator.h b/library/cpp/deprecated/split/split_iterator.h index 0eacc29228..73f3693564 100644 --- a/library/cpp/deprecated/split/split_iterator.h +++ b/library/cpp/deprecated/split/split_iterator.h @@ -4,74 +4,74 @@ #include <util/string/cast.h> #include <util/string/util.h> #include <util/string/builder.h> - -#include <util/system/yassert.h> + +#include <util/system/yassert.h> #include <util/system/defaults.h> #include <util/generic/strbuf.h> #include <util/generic/string.h> #include <util/generic/vector.h> #include <util/generic/yexception.h> -#include <cstdio> - -template <typename T> -struct TNumPair { - T Begin; - T End; - +#include <cstdio> + +template <typename T> +struct TNumPair { + T Begin; + T End; + TNumPair() = default; - - TNumPair(T begin, T end) - : Begin(begin) - , End(end) - { + + TNumPair(T begin, T end) + : Begin(begin) + , End(end) + { Y_ASSERT(begin <= end); - } - - T Length() const { - return End - Begin + 1; - } - - bool operator==(const TNumPair& r) const { - return (Begin == r.Begin) && (End == r.End); - } - - bool operator!=(const TNumPair& r) const { - return (Begin != r.Begin) || (End != r.End); - } -}; - + } + + T Length() const { + return End - Begin + 1; + } + + bool operator==(const TNumPair& r) const { + return (Begin == r.Begin) && (End == r.End); + } + + bool operator!=(const TNumPair& r) const { + return (Begin != r.Begin) || (End != r.End); + } +}; + using TSizeTRegion = TNumPair<size_t>; using TUi32Region = TNumPair<ui32>; - -template <> + +template <> inline TString ToString(const TUi32Region& r) { - return TStringBuilder() << "(" << r.Begin << ", " << r.End << ")"; -} - -template <> + return TStringBuilder() << "(" << r.Begin << ", " << r.End << ")"; +} + +template <> inline TUi32Region FromString(const TString& s) { - TUi32Region result; + TUi32Region result; sscanf(s.data(), "(%" PRIu32 ", %" PRIu32 ")", &result.Begin, &result.End); - return result; -} - -class TSplitDelimiters { + return result; +} + +class TSplitDelimiters { private: bool Delims[256]; public: explicit TSplitDelimiters(const char* s); - + Y_FORCE_INLINE bool IsDelimiter(ui8 ch) const { return Delims[ch]; } }; -template <class Split> +template <class Split> class TSplitIterator; -class TSplitBase { +class TSplitBase { protected: const char* Str; size_t Len; @@ -94,11 +94,11 @@ private: }; #ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable : 4512) +#pragma warning(push) +#pragma warning(disable : 4512) #endif -class TDelimitersSplit: public TSplitBase { +class TDelimitersSplit: public TSplitBase { private: const TSplitDelimiters& Delimiters; @@ -119,7 +119,7 @@ private: TDelimitersSplit(TString&&, const TSplitDelimiters&) = delete; }; -class TDelimitersStrictSplit: public TSplitBase { +class TDelimitersStrictSplit: public TSplitBase { private: const TSplitDelimiters& Delimiters; @@ -140,7 +140,7 @@ private: TDelimitersStrictSplit(TString&&, const TSplitDelimiters&) = delete; }; -class TScreenedDelimitersSplit: public TSplitBase { +class TScreenedDelimitersSplit: public TSplitBase { private: const TSplitDelimiters& Delimiters; const TSplitDelimiters& Screens; @@ -162,7 +162,7 @@ private: TScreenedDelimitersSplit(const TString&, const TSplitDelimiters&, TSplitDelimiters&&) = delete; }; -class TDelimitersSplitWithoutTags: public TSplitBase { +class TDelimitersSplitWithoutTags: public TSplitBase { private: const TSplitDelimiters& Delimiters; size_t SkipTag(size_t pos) const; @@ -185,7 +185,7 @@ private: TDelimitersSplitWithoutTags(TString&&, const TSplitDelimiters&) = delete; }; -class TCharSplit: public TSplitBase { +class TCharSplit: public TSplitBase { public: using TIterator = TSplitIterator<TCharSplit>; friend class TSplitIterator<TCharSplit>; @@ -202,10 +202,10 @@ private: }; #ifdef _MSC_VER -#pragma warning(pop) +#pragma warning(pop) #endif -class TCharSplitWithoutTags: public TSplitBase { +class TCharSplitWithoutTags: public TSplitBase { private: size_t SkipTag(size_t pos) const; size_t SkipDelimiters(size_t pos) const; @@ -225,7 +225,7 @@ private: TCharSplitWithoutTags(TString&&) = delete; }; -class TSubstringSplitDelimiter { +class TSubstringSplitDelimiter { public: TKMPMatcher Matcher; size_t Len; @@ -233,7 +233,7 @@ public: TSubstringSplitDelimiter(const TString& s); }; -class TSubstringSplit: public TSplitBase { +class TSubstringSplit: public TSplitBase { private: const TSubstringSplitDelimiter& Delimiter; @@ -253,8 +253,8 @@ private: TSubstringSplit(const TString&, TSubstringSplitDelimiter&&) = delete; }; -template <class TSplit> -class TSplitIterator { +template <class TSplit> +class TSplitIterator { protected: const TSplit& Split; size_t Pos; @@ -271,12 +271,12 @@ public: virtual ~TSplitIterator() { delete CurrentStroka; } - + inline TSizeTRegion Next() { Y_ENSURE(!Eof(), TStringBuf("eof reached")); return Split.Next(Pos); } - + TStringBuf NextTok() { if (Eof()) return TStringBuf(); @@ -308,8 +308,8 @@ public: using TSplitTokens = TVector<TString>; -template <typename TSplit> -void Split(const TSplit& split, TSplitTokens* words) { +template <typename TSplit> +void Split(const TSplit& split, TSplitTokens* words) { words->clear(); TSplitIterator<TSplit> it(split); while (!it.Eof()) diff --git a/library/cpp/deprecated/split/split_iterator_ut.cpp b/library/cpp/deprecated/split/split_iterator_ut.cpp index be5069c4be..c4df3b2d13 100644 --- a/library/cpp/deprecated/split/split_iterator_ut.cpp +++ b/library/cpp/deprecated/split/split_iterator_ut.cpp @@ -1,15 +1,15 @@ -#include "split_iterator.h" +#include "split_iterator.h" #include <library/cpp/testing/unittest/registar.h> -class TSplitIteratorTest: public TTestBase { +class TSplitIteratorTest: public TTestBase { UNIT_TEST_SUITE(TSplitIteratorTest); - UNIT_TEST(TestDelimiters); - UNIT_TEST(TestDelimitersSplit); - UNIT_TEST(TestDelimitersStrictSplit); - UNIT_TEST(TestTail); - UNIT_TEST(TestScreenedDelimitersSplit); - UNIT_TEST(TestSubstringDelimiter); + UNIT_TEST(TestDelimiters); + UNIT_TEST(TestDelimitersSplit); + UNIT_TEST(TestDelimitersStrictSplit); + UNIT_TEST(TestTail); + UNIT_TEST(TestScreenedDelimitersSplit); + UNIT_TEST(TestSubstringDelimiter); UNIT_TEST_SUITE_END(); public: @@ -21,17 +21,17 @@ public: void TestSubstringDelimiter(); }; -void TSplitIteratorTest::TestDelimiters() { +void TSplitIteratorTest::TestDelimiters() { TSplitDelimiters delims("@"); for (int i = 0; i < 256; ++i) - if ('@' != i) { + if ('@' != i) { UNIT_ASSERT(!delims.IsDelimiter((ui8)i)); - } else { + } else { UNIT_ASSERT(delims.IsDelimiter((ui8)i)); } } -void TSplitIteratorTest::TestDelimitersSplit() { +void TSplitIteratorTest::TestDelimitersSplit() { { TString s = "1a3b45cd"; TSplitDelimiters delims("abcd"); @@ -52,7 +52,7 @@ void TSplitIteratorTest::TestDelimitersSplit() { } } -void TSplitIteratorTest::TestDelimitersStrictSplit() { +void TSplitIteratorTest::TestDelimitersStrictSplit() { { TString s = "grp@2"; TSplitDelimiters delims("@"); @@ -74,7 +74,7 @@ void TSplitIteratorTest::TestDelimitersStrictSplit() { } } -void TSplitIteratorTest::TestTail() { +void TSplitIteratorTest::TestTail() { TString s = "grp@2@4"; TSplitDelimiters delims("@"); TDelimitersSplit split(s, delims); @@ -88,7 +88,7 @@ void TSplitIteratorTest::TestTail() { UNIT_ASSERT_EQUAL(it.GetTail(), ""); } -void TSplitIteratorTest::TestScreenedDelimitersSplit() { +void TSplitIteratorTest::TestScreenedDelimitersSplit() { { const TString s = "77.88.58.91 - - [28/Aug/2008:00:08:07 +0400] \"GET /export/mordashka.tgz HTTP/1.1\" 304 - \"-\" \"libwww-perl/5.805\" \"news.yandex.ru,80\" \"-\" \"-\" 1219867687 \"0\" 3283 2"; const TSplitDelimiters delims(" "); @@ -137,7 +137,7 @@ void TSplitIteratorTest::TestScreenedDelimitersSplit() { } } -void TSplitIteratorTest::TestSubstringDelimiter() { +void TSplitIteratorTest::TestSubstringDelimiter() { const TString s = "a@@bb@@cc@c.d@@r"; static const TSubstringSplitDelimiter delimiter("@@"); const TSubstringSplit splitter(s, delimiter); diff --git a/library/cpp/deprecated/ya.make b/library/cpp/deprecated/ya.make index 6c753f68a9..3fa78a0e1c 100644 --- a/library/cpp/deprecated/ya.make +++ b/library/cpp/deprecated/ya.make @@ -32,7 +32,7 @@ RECURSE( prog_options/ut sgi_hash threadable - transgene + transgene datawork datawork/conf calc_module diff --git a/library/cpp/diff/diff.cpp b/library/cpp/diff/diff.cpp index be57da7f39..658a269371 100644 --- a/library/cpp/diff/diff.cpp +++ b/library/cpp/diff/diff.cpp @@ -42,7 +42,7 @@ struct TCollection { template <> struct TCollection<char>: public TCollectionImpl<char> { - TCollection(const TStringBuf& str, const TString& delims) { + TCollection(const TStringBuf& str, const TString& delims) { TSetDelimiter<const char> set(delims.data()); TKeepDelimiters<TCollection<char>> c(this); SplitString(str.begin(), str.end(), set, c); @@ -51,7 +51,7 @@ struct TCollection<char>: public TCollectionImpl<char> { template <> struct TCollection<wchar16>: public TCollectionImpl<wchar16> { - TCollection(const TWtringBuf& str, const TUtf16String& delims) { + TCollection(const TWtringBuf& str, const TUtf16String& delims) { TSetDelimiter<const wchar16> set(delims.data()); TKeepDelimiters<TCollection<wchar16>> c(this); SplitString(str.begin(), str.end(), set, c); diff --git a/library/cpp/diff/diff.h b/library/cpp/diff/diff.h index 94fb00cd0b..9b6a199c72 100644 --- a/library/cpp/diff/diff.h +++ b/library/cpp/diff/diff.h @@ -109,4 +109,4 @@ namespace NDiff { size_t InlineDiff(TVector<TChunk<char>>& chunks, const TStringBuf& left, const TStringBuf& right, const TString& delims = TString()); size_t InlineDiff(TVector<TChunk<wchar16>>& chunks, const TWtringBuf& left, const TWtringBuf& right, const TUtf16String& delims = TUtf16String()); -} +} diff --git a/library/cpp/digest/argonish/ut/ut.cpp b/library/cpp/digest/argonish/ut/ut.cpp index 12ef530a18..67840aacc5 100644 --- a/library/cpp/digest/argonish/ut/ut.cpp +++ b/library/cpp/digest/argonish/ut/ut.cpp @@ -4,130 +4,130 @@ Y_UNIT_TEST_SUITE(ArgonishTest) { const ui8 GenKatPassword[32] = { - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, }; const ui8 GenKatSalt[16] = { - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, }; const ui8 GenKatSecret[8] = { - 0x03, - 0x03, - 0x03, - 0x03, - 0x03, - 0x03, - 0x03, - 0x03, + 0x03, + 0x03, + 0x03, + 0x03, + 0x03, + 0x03, + 0x03, + 0x03, }; const ui8 FrPassword[13] = { - 'm', - 'e', - 'g', - 'a', - '_', - 'p', - 'a', - 's', - 's', - 'w', - 'o', - 'r', - 'd', + 'm', + 'e', + 'g', + 'a', + '_', + 'p', + 'a', + 's', + 's', + 'w', + 'o', + 'r', + 'd', }; const ui8 FrSecret[16] = { - 'm', - 'e', - 'g', - 'a', - '_', - 's', - 'e', - 'c', - 'r', - 'e', - 't', - '_', - 'k', - 'e', - 'y', - '2', + 'm', + 'e', + 'g', + 'a', + '_', + 's', + 'e', + 'c', + 'r', + 'e', + 't', + '_', + 'k', + 'e', + 'y', + '2', }; const ui8 FrSalt[9] = { - 'm', - 'e', - 'g', - 'a', - '_', - 's', - 'a', - 'l', - 't', + 'm', + 'e', + 'g', + 'a', + '_', + 's', + 'a', + 'l', + 't', }; const ui8 GenKatAAD[12] = { - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, }; Y_UNIT_TEST(Argon2_Fr_Test) { diff --git a/library/cpp/digest/argonish/ut_fat/ut.cpp b/library/cpp/digest/argonish/ut_fat/ut.cpp index 41fa001685..faab88c7fc 100644 --- a/library/cpp/digest/argonish/ut_fat/ut.cpp +++ b/library/cpp/digest/argonish/ut_fat/ut.cpp @@ -4,83 +4,83 @@ Y_UNIT_TEST_SUITE(ArgonishTest) { const ui8 GenKatPassword[32] = { - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, - 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, }; const ui8 GenKatSalt[16] = { - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, - 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, + 0x02, }; const ui8 GenKatSecret[8] = { - 0x03, - 0x03, - 0x03, - 0x03, - 0x03, - 0x03, - 0x03, - 0x03, + 0x03, + 0x03, + 0x03, + 0x03, + 0x03, + 0x03, + 0x03, + 0x03, }; const ui8 GenKatAAD[12] = { - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, - 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, + 0x04, }; constexpr NArgonish::EInstructionSet MaxArch = diff --git a/library/cpp/digest/crc32c/crc32c.cpp b/library/cpp/digest/crc32c/crc32c.cpp index 369b46a213..63daa12859 100644 --- a/library/cpp/digest/crc32c/crc32c.cpp +++ b/library/cpp/digest/crc32c/crc32c.cpp @@ -1,40 +1,40 @@ #include "crc32c.h" - + #include <util/generic/singleton.h> #include <contrib/libs/crcutil/interface.h> -namespace { - typedef crcutil_interface::CRC TCrc; - - struct TCrcUtilSse4 { - TCrc* const Pimpl; +namespace { + typedef crcutil_interface::CRC TCrc; + + struct TCrcUtilSse4 { + TCrc* const Pimpl; TCrcUtilSse4() noexcept - : Pimpl(TCrc::Create(0x82f63b78, 0, 32, true, 0, 0, 0, TCrc::IsSSE42Available(), nullptr)) - { - } + : Pimpl(TCrc::Create(0x82f63b78, 0, 32, true, 0, 0, 0, TCrc::IsSSE42Available(), nullptr)) + { + } - ~TCrcUtilSse4() noexcept { - Pimpl->Delete(); - } + ~TCrcUtilSse4() noexcept { + Pimpl->Delete(); + } inline ui32 Extend(ui32 init, const void* data, size_t n) const noexcept { - crcutil_interface::UINT64 sum = init; - Pimpl->Compute(data, n, &sum); - return (ui32)sum; - } + crcutil_interface::UINT64 sum = init; + Pimpl->Compute(data, n, &sum); + return (ui32)sum; + } }; -} - -ui32 Crc32c(const void* p, size_t size) noexcept { - return Singleton<TCrcUtilSse4>()->Extend(0, p, size); -} - -ui32 Crc32cExtend(ui32 init, const void* data, size_t n) noexcept { - return Singleton<TCrcUtilSse4>()->Extend(init, data, n); -} +} -bool HaveFastCrc32c() noexcept { - return TCrc::IsSSE42Available(); +ui32 Crc32c(const void* p, size_t size) noexcept { + return Singleton<TCrcUtilSse4>()->Extend(0, p, size); } + +ui32 Crc32cExtend(ui32 init, const void* data, size_t n) noexcept { + return Singleton<TCrcUtilSse4>()->Extend(init, data, n); +} + +bool HaveFastCrc32c() noexcept { + return TCrc::IsSSE42Available(); +} diff --git a/library/cpp/digest/crc32c/crc32c.h b/library/cpp/digest/crc32c/crc32c.h index 17b554c8e1..2f4b190b9c 100644 --- a/library/cpp/digest/crc32c/crc32c.h +++ b/library/cpp/digest/crc32c/crc32c.h @@ -1,9 +1,9 @@ #pragma once -#include <util/system/types.h> +#include <util/system/types.h> // Threadsafe -ui32 Crc32c(const void* p, size_t size) noexcept; -ui32 Crc32cExtend(ui32 init, const void* data, size_t n) noexcept; - -bool HaveFastCrc32c() noexcept; +ui32 Crc32c(const void* p, size_t size) noexcept; +ui32 Crc32cExtend(ui32 init, const void* data, size_t n) noexcept; + +bool HaveFastCrc32c() noexcept; diff --git a/library/cpp/digest/crc32c/crc32c_ut.cpp b/library/cpp/digest/crc32c/crc32c_ut.cpp index aa31b83422..08e7bbf120 100644 --- a/library/cpp/digest/crc32c/crc32c_ut.cpp +++ b/library/cpp/digest/crc32c/crc32c_ut.cpp @@ -1,12 +1,12 @@ -#include "crc32c.h" - +#include "crc32c.h" + #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TestCrc32c) { Y_UNIT_TEST(TestCalc) { - UNIT_ASSERT_VALUES_EQUAL(Crc32c("abc", 3), ui32(910901175)); - } - + UNIT_ASSERT_VALUES_EQUAL(Crc32c("abc", 3), ui32(910901175)); + } + Y_UNIT_TEST(TestUnaligned) { const TString str(1000, 'a'); for (size_t substrLen = 0; substrLen <= str.length(); ++substrLen) { @@ -18,6 +18,6 @@ Y_UNIT_TEST_SUITE(TestCrc32c) { } Y_UNIT_TEST(TestExtend) { - UNIT_ASSERT_VALUES_EQUAL(Crc32cExtend(1, "abc", 3), ui32(2466950601)); - } -} + UNIT_ASSERT_VALUES_EQUAL(Crc32cExtend(1, "abc", 3), ui32(2466950601)); + } +} diff --git a/library/cpp/digest/crc32c/ut/ya.make b/library/cpp/digest/crc32c/ut/ya.make index 325622e080..8ce179aaa8 100644 --- a/library/cpp/digest/crc32c/ut/ya.make +++ b/library/cpp/digest/crc32c/ut/ya.make @@ -1,9 +1,9 @@ UNITTEST_FOR(library/cpp/digest/crc32c) - -OWNER(pg) - -SRCS( - crc32c_ut.cpp -) - -END() + +OWNER(pg) + +SRCS( + crc32c_ut.cpp +) + +END() diff --git a/library/cpp/digest/crc32c/ya.make b/library/cpp/digest/crc32c/ya.make index d6faf16c9c..5d21e907a0 100644 --- a/library/cpp/digest/crc32c/ya.make +++ b/library/cpp/digest/crc32c/ya.make @@ -1,20 +1,20 @@ LIBRARY() -#!!! +#!!! OWNER( ddoarn a-romanov - pg - gulin - dcherednik + pg + gulin + dcherednik g:yamr g:rtmr ) -PEERDIR( - contrib/libs/crcutil -) - +PEERDIR( + contrib/libs/crcutil +) + SRCS( crc32c.cpp ) diff --git a/library/cpp/digest/lower_case/hash_ops.cpp b/library/cpp/digest/lower_case/hash_ops.cpp index 3f1746d830..cbc25394a8 100644 --- a/library/cpp/digest/lower_case/hash_ops.cpp +++ b/library/cpp/digest/lower_case/hash_ops.cpp @@ -1,23 +1,23 @@ -#include "lchash.h" -#include "lciter.h" -#include "hash_ops.h" - -#include <util/generic/algorithm.h> - -size_t TCIOps::operator()(const TStringBuf& s) const noexcept { - return FnvCaseLess(s, (size_t)0xBEE); -} - -size_t TCIOps::operator()(const char* s) const noexcept { - return operator()(TStringBuf(s)); -} - -bool TCIOps::operator()(const TStringBuf& f, const TStringBuf& s) const noexcept { - using TIter = TLowerCaseIterator<const char>; - +#include "lchash.h" +#include "lciter.h" +#include "hash_ops.h" + +#include <util/generic/algorithm.h> + +size_t TCIOps::operator()(const TStringBuf& s) const noexcept { + return FnvCaseLess(s, (size_t)0xBEE); +} + +size_t TCIOps::operator()(const char* s) const noexcept { + return operator()(TStringBuf(s)); +} + +bool TCIOps::operator()(const TStringBuf& f, const TStringBuf& s) const noexcept { + using TIter = TLowerCaseIterator<const char>; + return (f.size() == s.size()) && Equal(TIter(f.begin()), TIter(f.end()), TIter(s.begin())); -} - -bool TCIOps::operator()(const char* f, const char* s) const noexcept { - return operator()(TStringBuf(f), TStringBuf(s)); -} +} + +bool TCIOps::operator()(const char* f, const char* s) const noexcept { + return operator()(TStringBuf(f), TStringBuf(s)); +} diff --git a/library/cpp/digest/lower_case/hash_ops.h b/library/cpp/digest/lower_case/hash_ops.h index 83ebf7aca4..54267c1ee4 100644 --- a/library/cpp/digest/lower_case/hash_ops.h +++ b/library/cpp/digest/lower_case/hash_ops.h @@ -1,13 +1,13 @@ -#pragma once - -#include <util/generic/strbuf.h> - +#pragma once + +#include <util/generic/strbuf.h> + // can be used for caseless hashes like: THashMap<TStringBuf, T, TCIOps, TCIOps> - -struct TCIOps { - size_t operator()(const char* s) const noexcept; - size_t operator()(const TStringBuf& s) const noexcept; - - bool operator()(const char* f, const char* s) const noexcept; - bool operator()(const TStringBuf& f, const TStringBuf& s) const noexcept; -}; + +struct TCIOps { + size_t operator()(const char* s) const noexcept; + size_t operator()(const TStringBuf& s) const noexcept; + + bool operator()(const char* f, const char* s) const noexcept; + bool operator()(const TStringBuf& f, const TStringBuf& s) const noexcept; +}; diff --git a/library/cpp/digest/lower_case/hash_ops_ut.cpp b/library/cpp/digest/lower_case/hash_ops_ut.cpp index a7ab0b86ea..47f154ffb2 100644 --- a/library/cpp/digest/lower_case/hash_ops_ut.cpp +++ b/library/cpp/digest/lower_case/hash_ops_ut.cpp @@ -1,36 +1,36 @@ -#include "hash_ops.h" - +#include "hash_ops.h" + #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TestCIHash) { Y_UNIT_TEST(TestYHash1) { THashMap<TStringBuf, int, TCIOps, TCIOps> h; - - h["Ab"] = 1; - h["aB"] = 2; - - UNIT_ASSERT_VALUES_EQUAL(h.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(h["ab"], 2); - } - + + h["Ab"] = 1; + h["aB"] = 2; + + UNIT_ASSERT_VALUES_EQUAL(h.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(h["ab"], 2); + } + Y_UNIT_TEST(TestYHash2) { THashMap<const char*, int, TCIOps, TCIOps> h; - - h["Ab"] = 1; - h["aB"] = 2; - - UNIT_ASSERT_VALUES_EQUAL(h.size(), 1); - UNIT_ASSERT_VALUES_EQUAL(h["ab"], 2); - - h["Bc"] = 2; - h["bC"] = 3; - - UNIT_ASSERT_VALUES_EQUAL(h.size(), 2); - UNIT_ASSERT_VALUES_EQUAL(h["bc"], 3); - } - + + h["Ab"] = 1; + h["aB"] = 2; + + UNIT_ASSERT_VALUES_EQUAL(h.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(h["ab"], 2); + + h["Bc"] = 2; + h["bC"] = 3; + + UNIT_ASSERT_VALUES_EQUAL(h.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(h["bc"], 3); + } + Y_UNIT_TEST(Test1) { UNIT_ASSERT_VALUES_EQUAL(TCIOps()("aBc3"), TCIOps()(TStringBuf("AbC3"))); - UNIT_ASSERT(TCIOps()("aBc4", "AbC4")); - } -} + UNIT_ASSERT(TCIOps()("aBc4", "AbC4")); + } +} diff --git a/library/cpp/digest/lower_case/lchash.cpp b/library/cpp/digest/lower_case/lchash.cpp index 92b4245533..51a7ff5799 100644 --- a/library/cpp/digest/lower_case/lchash.cpp +++ b/library/cpp/digest/lower_case/lchash.cpp @@ -1 +1 @@ -#include "lchash.h" +#include "lchash.h" diff --git a/library/cpp/digest/lower_case/lchash.h b/library/cpp/digest/lower_case/lchash.h index 6a287d9479..007d723e21 100644 --- a/library/cpp/digest/lower_case/lchash.h +++ b/library/cpp/digest/lower_case/lchash.h @@ -1,18 +1,18 @@ -#pragma once - -#include "lciter.h" - -#include <util/digest/fnv.h> -#include <util/generic/strbuf.h> - -template <class T> +#pragma once + +#include "lciter.h" + +#include <util/digest/fnv.h> +#include <util/generic/strbuf.h> + +template <class T> static inline T FnvCaseLess(const char* b, size_t l, T t = 0) noexcept { using TIter = TLowerCaseIterator<const char>; - - return FnvHash(TIter(b), TIter(b + l), t); -} - -template <class T> + + return FnvHash(TIter(b), TIter(b + l), t); +} + +template <class T> static inline T FnvCaseLess(const TStringBuf& s, T t = 0) noexcept { return FnvCaseLess(s.data(), s.size(), t); -} +} diff --git a/library/cpp/digest/lower_case/lchash_ut.cpp b/library/cpp/digest/lower_case/lchash_ut.cpp index 5711fe7cd7..9b40f38667 100644 --- a/library/cpp/digest/lower_case/lchash_ut.cpp +++ b/library/cpp/digest/lower_case/lchash_ut.cpp @@ -1,21 +1,21 @@ -#include "lchash.h" - +#include "lchash.h" + #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TWebDaemonHash) { Y_UNIT_TEST(Stability) { - UNIT_ASSERT_VALUES_EQUAL(FnvCaseLess<ui64>("blah"), ULL(5923727754379976229)); - UNIT_ASSERT_VALUES_EQUAL(FnvCaseLess<ui64>("blahminor"), ULL(8755704309003440816)); - } - + UNIT_ASSERT_VALUES_EQUAL(FnvCaseLess<ui64>("blah"), ULL(5923727754379976229)); + UNIT_ASSERT_VALUES_EQUAL(FnvCaseLess<ui64>("blahminor"), ULL(8755704309003440816)); + } + Y_UNIT_TEST(CaseLess) { - UNIT_ASSERT_VALUES_EQUAL(FnvCaseLess<ui64>("blah"), FnvCaseLess<ui64>("bLah")); - UNIT_ASSERT_VALUES_EQUAL(FnvCaseLess<ui64>("blah"), FnvCaseLess<ui64>("blAh")); - UNIT_ASSERT_VALUES_EQUAL(FnvCaseLess<ui64>("blah"), FnvCaseLess<ui64>("BLAH")); - } - + UNIT_ASSERT_VALUES_EQUAL(FnvCaseLess<ui64>("blah"), FnvCaseLess<ui64>("bLah")); + UNIT_ASSERT_VALUES_EQUAL(FnvCaseLess<ui64>("blah"), FnvCaseLess<ui64>("blAh")); + UNIT_ASSERT_VALUES_EQUAL(FnvCaseLess<ui64>("blah"), FnvCaseLess<ui64>("BLAH")); + } + Y_UNIT_TEST(Robustness) { - UNIT_ASSERT(FnvCaseLess<ui64>("x-real-ip") != FnvCaseLess<ui64>("x-req-id")); - UNIT_ASSERT(FnvCaseLess<ui64>("x-real-ip") != FnvCaseLess<ui64>("x-start-time")); - } -} + UNIT_ASSERT(FnvCaseLess<ui64>("x-real-ip") != FnvCaseLess<ui64>("x-req-id")); + UNIT_ASSERT(FnvCaseLess<ui64>("x-real-ip") != FnvCaseLess<ui64>("x-start-time")); + } +} diff --git a/library/cpp/digest/lower_case/lciter.cpp b/library/cpp/digest/lower_case/lciter.cpp index 38d23eb2dd..d204536e8d 100644 --- a/library/cpp/digest/lower_case/lciter.cpp +++ b/library/cpp/digest/lower_case/lciter.cpp @@ -1 +1 @@ -#include "lciter.h" +#include "lciter.h" diff --git a/library/cpp/digest/lower_case/lciter.h b/library/cpp/digest/lower_case/lciter.h index 9538e57932..87f47c7a82 100644 --- a/library/cpp/digest/lower_case/lciter.h +++ b/library/cpp/digest/lower_case/lciter.h @@ -1,42 +1,42 @@ -#pragma once - -#include <util/generic/typetraits.h> -#include <util/string/ascii.h> - +#pragma once + +#include <util/generic/typetraits.h> +#include <util/string/ascii.h> + #include <iterator> - -template <class T> + +template <class T> struct TLowerCaseIterator: public std::iterator<std::input_iterator_tag, T> { using TNonConst = std::remove_const_t<T>; - - inline TLowerCaseIterator(T* c) - : C(c) - { - } - + + inline TLowerCaseIterator(T* c) + : C(c) + { + } + inline TLowerCaseIterator& operator++() noexcept { - ++C; - - return *this; - } - + ++C; + + return *this; + } + inline TLowerCaseIterator operator++(int) noexcept { - return C++; - } - + return C++; + } + inline TNonConst operator*() const noexcept { - return AsciiToLower(*C); - } - - T* C; -}; - -template <class T> + return AsciiToLower(*C); + } + + T* C; +}; + +template <class T> inline bool operator==(const TLowerCaseIterator<T>& l, const TLowerCaseIterator<T>& r) noexcept { - return l.C == r.C; -} - -template <class T> + return l.C == r.C; +} + +template <class T> inline bool operator!=(const TLowerCaseIterator<T>& l, const TLowerCaseIterator<T>& r) noexcept { - return !(l == r); -} + return !(l == r); +} diff --git a/library/cpp/digest/lower_case/ut/ya.make b/library/cpp/digest/lower_case/ut/ya.make index f083257b3d..adfb14818d 100644 --- a/library/cpp/digest/lower_case/ut/ya.make +++ b/library/cpp/digest/lower_case/ut/ya.make @@ -1,13 +1,13 @@ UNITTEST_FOR(library/cpp/digest/lower_case) - + OWNER( pg g:util ) - -SRCS( - lchash_ut.cpp - hash_ops_ut.cpp -) - -END() + +SRCS( + lchash_ut.cpp + hash_ops_ut.cpp +) + +END() diff --git a/library/cpp/digest/lower_case/ya.make b/library/cpp/digest/lower_case/ya.make index 62cc16556a..bb6dee8ea9 100644 --- a/library/cpp/digest/lower_case/ya.make +++ b/library/cpp/digest/lower_case/ya.make @@ -1,14 +1,14 @@ -LIBRARY() - -OWNER( - pg +LIBRARY() + +OWNER( + pg g:util -) - -SRCS( - lciter.cpp - lchash.cpp - hash_ops.cpp -) - -END() +) + +SRCS( + lciter.cpp + lchash.cpp + hash_ops.cpp +) + +END() diff --git a/library/cpp/digest/md5/bench/main.cpp b/library/cpp/digest/md5/bench/main.cpp index 70672e6741..a987eefe59 100644 --- a/library/cpp/digest/md5/bench/main.cpp +++ b/library/cpp/digest/md5/bench/main.cpp @@ -1,19 +1,19 @@ #include <benchmark/benchmark.h> #include <library/cpp/digest/md5/md5.h> - -#define MD5_DEF(N) \ + +#define MD5_DEF(N) \ static void MD5Benchmark_##N(benchmark::State& st) { \ - char buf[N]; \ + char buf[N]; \ for (auto _ : st) { \ - Y_DO_NOT_OPTIMIZE_AWAY(MD5().Update(buf, sizeof(buf))); \ - } \ + Y_DO_NOT_OPTIMIZE_AWAY(MD5().Update(buf, sizeof(buf))); \ + } \ } \ BENCHMARK(MD5Benchmark_##N); - -MD5_DEF(32) -MD5_DEF(64) -MD5_DEF(128) - -MD5_DEF(1024) -MD5_DEF(2048) + +MD5_DEF(32) +MD5_DEF(64) +MD5_DEF(128) + +MD5_DEF(1024) +MD5_DEF(2048) diff --git a/library/cpp/digest/md5/bench/ya.make b/library/cpp/digest/md5/bench/ya.make index 5c3e3c1c42..0c19f330a0 100644 --- a/library/cpp/digest/md5/bench/ya.make +++ b/library/cpp/digest/md5/bench/ya.make @@ -1,16 +1,16 @@ G_BENCHMARK() - + OWNER( pg g:util ) - -PEERDIR( + +PEERDIR( library/cpp/digest/md5 -) - -SRCS( - main.cpp -) - -END() +) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/digest/md5/md5.cpp b/library/cpp/digest/md5/md5.cpp index 24a5b69eef..4246270741 100644 --- a/library/cpp/digest/md5/md5.cpp +++ b/library/cpp/digest/md5/md5.cpp @@ -1,31 +1,31 @@ #include "md5.h" #include <library/cpp/string_utils/base64/base64.h> - + #include <util/stream/input.h> -#include <util/stream/file.h> +#include <util/stream/file.h> #include <util/string/hex.h> #include <contrib/libs/nayuki_md5/md5.h> - -namespace { + +namespace { constexpr size_t MD5_BLOCK_LENGTH = 64; constexpr size_t MD5_PADDING_SHIFT = 56; constexpr size_t MD5_HEX_DIGEST_LENGTH = 32; struct TMd5Stream: public IOutputStream { - inline TMd5Stream(MD5* md5) - : M_(md5) - { - } - + inline TMd5Stream(MD5* md5) + : M_(md5) + { + } + void DoWrite(const void* buf, size_t len) override { - M_->Update(buf, len); - } - - MD5* M_; - }; + M_->Update(buf, len); + } + + MD5* M_; + }; inline TArrayRef<const ui8> MakeUnsignedArrayRef(const void* data, const size_t size) { return MakeArrayRef(static_cast<const ui8*>(data), size); @@ -34,16 +34,16 @@ namespace { inline TArrayRef<const ui8> MakeUnsignedArrayRef(const TArrayRef<const char>& data) { return MakeUnsignedArrayRef(data.data(), data.size()); } -} - -char* MD5::File(const char* filename, char* buf) { - try { +} + +char* MD5::File(const char* filename, char* buf) { + try { TUnbufferedFileInput fi(filename); - - return Stream(&fi, buf); - } catch (...) { + + return Stream(&fi, buf); + } catch (...) { } - + return nullptr; } @@ -77,25 +77,25 @@ TString MD5::Data(TStringBuf data) { } char* MD5::Stream(IInputStream* in, char* buf) { - return MD5().Update(in).End(buf); -} - + return MD5().Update(in).End(buf); +} + MD5& MD5::Update(IInputStream* in) { - TMd5Stream md5(this); - + TMd5Stream md5(this); + TransferData(in, &md5); - - return *this; + + return *this; } static const ui8 PADDING[MD5_BLOCK_LENGTH] = { - 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; /* MD5 initialization. Begins an MD5 operation, writing a new context. */ -void MD5::Init() { +void MD5::Init() { BufferSize = 0; StreamSize = 0; /* Load magic initialization constants. */ @@ -105,7 +105,7 @@ void MD5::Init() { State[3] = 0x10325476; } -/* +/* * MD5 block update operation. Continues an MD5 message-digest * operation, processing another message block, and updating the * context. @@ -141,7 +141,7 @@ void MD5::UpdatePart(TArrayRef<const ui8> data) { * MD5 padding. Adds padding followed by original length. */ -void MD5::Pad() { +void MD5::Pad() { size_t streamSize = StreamSize; const size_t paddingSize = (MD5_PADDING_SHIFT > BufferSize) ? (MD5_PADDING_SHIFT - BufferSize) : (MD5_PADDING_SHIFT + MD5_BLOCK_LENGTH - BufferSize); @@ -169,16 +169,16 @@ ui8* MD5::Final(ui8 digest[16]) { memcpy(digest, State, 16); /* Zeroize sensitive information. */ Init(); - - return digest; + + return digest; } -char* MD5::End(char* buf) { - static const char hex[] = "0123456789abcdef"; +char* MD5::End(char* buf) { + static const char hex[] = "0123456789abcdef"; ui8 digest[16]; - if (!buf) + if (!buf) buf = (char*)malloc(33); - if (!buf) + if (!buf) return nullptr; Final(digest); for (ui8 i = 0; i < MD5_HEX_DIGEST_LENGTH / 2; i++) { @@ -189,14 +189,14 @@ char* MD5::End(char* buf) { return buf; } -char* MD5::End_b64(char* buf) { +char* MD5::End_b64(char* buf) { ui8 digest[16]; - if (!buf) + if (!buf) buf = (char*)malloc(25); - if (!buf) + if (!buf) return nullptr; Final(digest); - Base64Encode(buf, digest, 16); + Base64Encode(buf, digest, 16); buf[24] = '\0'; return buf; } @@ -215,11 +215,11 @@ ui64 MD5::EndHalfMix() { TString MD5::Calc(TStringBuf data) { return Calc(MakeUnsignedArrayRef(data)); } - + TString MD5::Calc(const TArrayRef<const ui8>& data) { return Data(data); } - + TString MD5::CalcRaw(TStringBuf data) { return CalcRaw(MakeUnsignedArrayRef(data)); } diff --git a/library/cpp/digest/md5/md5.h b/library/cpp/digest/md5/md5.h index 2c17aa0518..014323adea 100644 --- a/library/cpp/digest/md5/md5.h +++ b/library/cpp/digest/md5/md5.h @@ -5,30 +5,30 @@ class IInputStream; -class MD5 { +class MD5 { public: MD5() { Init(); } - - void Init(); - + + void Init(); + inline MD5& Update(const void* data, size_t len) { return Update(MakeArrayRef(static_cast<const ui8*>(data), len)); } - + inline MD5& Update(TStringBuf data) { return Update(data.data(), data.size()); } - + inline MD5& Update(const TArrayRef<const ui8> data) { UpdatePart(data); return *this; - } - - void Pad(); + } + + void Pad(); ui8* Final(ui8[16]); - + // buf must be char[33]; char* End(char* buf); @@ -39,7 +39,7 @@ public: ui64 EndHalfMix(); MD5& Update(IInputStream* in); - + /* * Return hex-encoded md5 checksum for given file. * @@ -53,7 +53,7 @@ public: static TString Data(const TArrayRef<const ui8>& data); static TString Data(TStringBuf data); static char* Stream(IInputStream* in, char* buf); - + static TString Calc(TStringBuf data); // 32-byte hex-encoded static TString Calc(const TArrayRef<const ui8>& data); // 32-byte hex-encoded static TString CalcRaw(TStringBuf data); // 16-byte raw diff --git a/library/cpp/digest/md5/md5_medium_ut.cpp b/library/cpp/digest/md5/md5_medium_ut.cpp index a940c5cb66..11a8a1276c 100644 --- a/library/cpp/digest/md5/md5_medium_ut.cpp +++ b/library/cpp/digest/md5/md5_medium_ut.cpp @@ -1,24 +1,24 @@ -#include "md5.h" - +#include "md5.h" + #include <library/cpp/testing/unittest/registar.h> Y_UNIT_TEST_SUITE(TMD5MediumTest) { Y_UNIT_TEST(TestOverflow) { - if (sizeof(size_t) > sizeof(unsigned int)) { + if (sizeof(size_t) > sizeof(unsigned int)) { const size_t maxUi32 = (size_t)Max<unsigned int>(); TArrayHolder<char> buf(new char[maxUi32]); - + memset(buf.Get(), 0, maxUi32); - + MD5 r; for (int i = 0; i < 5; ++i) { r.Update(buf.Get(), maxUi32); } - + char rs[33]; TString s(r.End(rs)); s.to_lower(); - + UNIT_ASSERT_VALUES_EQUAL(s, "34a5a7ed4f0221310084e21a1e599659"); } } diff --git a/library/cpp/digest/md5/md5_ut.cpp b/library/cpp/digest/md5/md5_ut.cpp index 1c3e4ad0a9..8216fa463a 100644 --- a/library/cpp/digest/md5/md5_ut.cpp +++ b/library/cpp/digest/md5/md5_ut.cpp @@ -1,44 +1,44 @@ -#include "md5.h" - +#include "md5.h" + #include <library/cpp/testing/unittest/registar.h> -#include <util/system/fs.h> -#include <util/stream/file.h> - +#include <util/system/fs.h> +#include <util/stream/file.h> + Y_UNIT_TEST_SUITE(TMD5Test) { Y_UNIT_TEST(TestMD5) { // echo -n 'qwertyuiopqwertyuiopasdfghjklasdfghjkl' | md5sum constexpr const char* b = "qwertyuiopqwertyuiopasdfghjklasdfghjkl"; - + MD5 r; r.Update((const unsigned char*)b, 15); r.Update((const unsigned char*)b + 15, strlen(b) - 15); - + char rs[33]; TString s(r.End(rs)); s.to_lower(); - + UNIT_ASSERT_NO_DIFF(s, TStringBuf("3ac00dd696b966fd74deee3c35a59d8f")); - + TString result = r.Calc(TStringBuf(b)); result.to_lower(); UNIT_ASSERT_NO_DIFF(result, TStringBuf("3ac00dd696b966fd74deee3c35a59d8f")); } - + Y_UNIT_TEST(TestFile) { TString s = NUnitTest::RandomString(1000000, 1); const TString tmpFile = "tmp"; - + { TFixedBufferFileOutput fo(tmpFile); fo.Write(s.data(), s.size()); } - + char fileBuf[100]; char memBuf[100]; TString fileHash = MD5::File(tmpFile.data(), fileBuf); TString memoryHash = MD5::Data((const unsigned char*)s.data(), s.size(), memBuf); - + UNIT_ASSERT_NO_DIFF(fileHash, memoryHash); fileHash = MD5::File(tmpFile); diff --git a/library/cpp/digest/md5/medium_ut/ya.make b/library/cpp/digest/md5/medium_ut/ya.make index 418c57f086..5d4f2bbce7 100644 --- a/library/cpp/digest/md5/medium_ut/ya.make +++ b/library/cpp/digest/md5/medium_ut/ya.make @@ -1,5 +1,5 @@ UNITTEST_FOR(library/cpp/digest/md5) - + SIZE(MEDIUM) TIMEOUT(120) @@ -8,11 +8,11 @@ OWNER( pg g:util ) - -SRCS( + +SRCS( md5_medium_ut.cpp -) - +) + REQUIREMENTS(ram:10) -END() +END() diff --git a/library/cpp/digest/md5/ut/ya.make b/library/cpp/digest/md5/ut/ya.make index ad1eddbff2..d4595d71fd 100644 --- a/library/cpp/digest/md5/ut/ya.make +++ b/library/cpp/digest/md5/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/digest/md5) - + OWNER( pg g:util ) - -SRCS( - md5_ut.cpp -) - -END() + +SRCS( + md5_ut.cpp +) + +END() diff --git a/library/cpp/digest/md5/ya.make b/library/cpp/digest/md5/ya.make index c09ec1c326..501929c448 100644 --- a/library/cpp/digest/md5/ya.make +++ b/library/cpp/digest/md5/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - +LIBRARY() + OWNER( pg g:util ) - -SRCS( - md5.cpp -) - + +SRCS( + md5.cpp +) + PEERDIR( - contrib/libs/nayuki_md5 + contrib/libs/nayuki_md5 library/cpp/string_utils/base64 ) -END() +END() diff --git a/library/cpp/digest/old_crc/crc.cpp b/library/cpp/digest/old_crc/crc.cpp index 994755f34d..08032a89db 100644 --- a/library/cpp/digest/old_crc/crc.cpp +++ b/library/cpp/digest/old_crc/crc.cpp @@ -1,268 +1,268 @@ -#include "crc.h" +#include "crc.h" #include <library/cpp/digest/old_crc/crc.inc> -#include <util/system/defaults.h> - +#include <util/system/defaults.h> + static const ui64 CRCTAB64[256] = { - ULL(0x0000000000000000), - ULL(0xE543279765927881), - ULL(0x2FC568B9AEB68983), - ULL(0xCA864F2ECB24F102), - ULL(0x5F8AD1735D6D1306), - ULL(0xBAC9F6E438FF6B87), - ULL(0x704FB9CAF3DB9A85), - ULL(0x950C9E5D9649E204), - ULL(0xBF15A2E6BADA260C), - ULL(0x5A568571DF485E8D), - ULL(0x90D0CA5F146CAF8F), - ULL(0x7593EDC871FED70E), - ULL(0xE09F7395E7B7350A), - ULL(0x05DC540282254D8B), - ULL(0xCF5A1B2C4901BC89), - ULL(0x2A193CBB2C93C408), - ULL(0x9B68625A10263499), - ULL(0x7E2B45CD75B44C18), - ULL(0xB4AD0AE3BE90BD1A), - ULL(0x51EE2D74DB02C59B), - ULL(0xC4E2B3294D4B279F), - ULL(0x21A194BE28D95F1E), - ULL(0xEB27DB90E3FDAE1C), - ULL(0x0E64FC07866FD69D), - ULL(0x247DC0BCAAFC1295), - ULL(0xC13EE72BCF6E6A14), - ULL(0x0BB8A805044A9B16), - ULL(0xEEFB8F9261D8E397), - ULL(0x7BF711CFF7910193), - ULL(0x9EB4365892037912), - ULL(0x5432797659278810), - ULL(0xB1715EE13CB5F091), - ULL(0xD393E32345DE11B3), - ULL(0x36D0C4B4204C6932), - ULL(0xFC568B9AEB689830), - ULL(0x1915AC0D8EFAE0B1), - ULL(0x8C19325018B302B5), - ULL(0x695A15C77D217A34), - ULL(0xA3DC5AE9B6058B36), - ULL(0x469F7D7ED397F3B7), - ULL(0x6C8641C5FF0437BF), - ULL(0x89C566529A964F3E), - ULL(0x4343297C51B2BE3C), - ULL(0xA6000EEB3420C6BD), - ULL(0x330C90B6A26924B9), - ULL(0xD64FB721C7FB5C38), - ULL(0x1CC9F80F0CDFAD3A), - ULL(0xF98ADF98694DD5BB), - ULL(0x48FB817955F8252A), - ULL(0xADB8A6EE306A5DAB), - ULL(0x673EE9C0FB4EACA9), - ULL(0x827DCE579EDCD428), - ULL(0x1771500A0895362C), - ULL(0xF232779D6D074EAD), - ULL(0x38B438B3A623BFAF), - ULL(0xDDF71F24C3B1C72E), - ULL(0xF7EE239FEF220326), - ULL(0x12AD04088AB07BA7), - ULL(0xD82B4B2641948AA5), - ULL(0x3D686CB12406F224), - ULL(0xA864F2ECB24F1020), - ULL(0x4D27D57BD7DD68A1), - ULL(0x87A19A551CF999A3), - ULL(0x62E2BDC2796BE122), - ULL(0x4264E1D1EE2E5BE7), - ULL(0xA727C6468BBC2366), - ULL(0x6DA189684098D264), - ULL(0x88E2AEFF250AAAE5), - ULL(0x1DEE30A2B34348E1), - ULL(0xF8AD1735D6D13060), - ULL(0x322B581B1DF5C162), - ULL(0xD7687F8C7867B9E3), - ULL(0xFD71433754F47DEB), - ULL(0x183264A03166056A), - ULL(0xD2B42B8EFA42F468), - ULL(0x37F70C199FD08CE9), - ULL(0xA2FB924409996EED), - ULL(0x47B8B5D36C0B166C), - ULL(0x8D3EFAFDA72FE76E), - ULL(0x687DDD6AC2BD9FEF), - ULL(0xD90C838BFE086F7E), - ULL(0x3C4FA41C9B9A17FF), - ULL(0xF6C9EB3250BEE6FD), - ULL(0x138ACCA5352C9E7C), - ULL(0x868652F8A3657C78), - ULL(0x63C5756FC6F704F9), - ULL(0xA9433A410DD3F5FB), - ULL(0x4C001DD668418D7A), - ULL(0x6619216D44D24972), - ULL(0x835A06FA214031F3), - ULL(0x49DC49D4EA64C0F1), - ULL(0xAC9F6E438FF6B870), - ULL(0x3993F01E19BF5A74), - ULL(0xDCD0D7897C2D22F5), - ULL(0x165698A7B709D3F7), - ULL(0xF315BF30D29BAB76), - ULL(0x91F702F2ABF04A54), - ULL(0x74B42565CE6232D5), - ULL(0xBE326A4B0546C3D7), - ULL(0x5B714DDC60D4BB56), - ULL(0xCE7DD381F69D5952), - ULL(0x2B3EF416930F21D3), - ULL(0xE1B8BB38582BD0D1), - ULL(0x04FB9CAF3DB9A850), - ULL(0x2EE2A014112A6C58), - ULL(0xCBA1878374B814D9), - ULL(0x0127C8ADBF9CE5DB), - ULL(0xE464EF3ADA0E9D5A), - ULL(0x716871674C477F5E), - ULL(0x942B56F029D507DF), - ULL(0x5EAD19DEE2F1F6DD), - ULL(0xBBEE3E4987638E5C), - ULL(0x0A9F60A8BBD67ECD), - ULL(0xEFDC473FDE44064C), - ULL(0x255A08111560F74E), - ULL(0xC0192F8670F28FCF), - ULL(0x5515B1DBE6BB6DCB), - ULL(0xB056964C8329154A), - ULL(0x7AD0D962480DE448), - ULL(0x9F93FEF52D9F9CC9), - ULL(0xB58AC24E010C58C1), - ULL(0x50C9E5D9649E2040), - ULL(0x9A4FAAF7AFBAD142), - ULL(0x7F0C8D60CA28A9C3), - ULL(0xEA00133D5C614BC7), - ULL(0x0F4334AA39F33346), - ULL(0xC5C57B84F2D7C244), - ULL(0x20865C139745BAC5), - ULL(0x84C9C3A3DC5CB7CE), - ULL(0x618AE434B9CECF4F), - ULL(0xAB0CAB1A72EA3E4D), - ULL(0x4E4F8C8D177846CC), - ULL(0xDB4312D08131A4C8), - ULL(0x3E003547E4A3DC49), - ULL(0xF4867A692F872D4B), - ULL(0x11C55DFE4A1555CA), - ULL(0x3BDC6145668691C2), - ULL(0xDE9F46D20314E943), - ULL(0x141909FCC8301841), - ULL(0xF15A2E6BADA260C0), - ULL(0x6456B0363BEB82C4), - ULL(0x811597A15E79FA45), - ULL(0x4B93D88F955D0B47), - ULL(0xAED0FF18F0CF73C6), - ULL(0x1FA1A1F9CC7A8357), - ULL(0xFAE2866EA9E8FBD6), - ULL(0x3064C94062CC0AD4), - ULL(0xD527EED7075E7255), - ULL(0x402B708A91179051), - ULL(0xA568571DF485E8D0), - ULL(0x6FEE18333FA119D2), - ULL(0x8AAD3FA45A336153), - ULL(0xA0B4031F76A0A55B), - ULL(0x45F724881332DDDA), - ULL(0x8F716BA6D8162CD8), - ULL(0x6A324C31BD845459), - ULL(0xFF3ED26C2BCDB65D), - ULL(0x1A7DF5FB4E5FCEDC), - ULL(0xD0FBBAD5857B3FDE), - ULL(0x35B89D42E0E9475F), - ULL(0x575A20809982A67D), - ULL(0xB2190717FC10DEFC), - ULL(0x789F483937342FFE), - ULL(0x9DDC6FAE52A6577F), - ULL(0x08D0F1F3C4EFB57B), - ULL(0xED93D664A17DCDFA), - ULL(0x2715994A6A593CF8), - ULL(0xC256BEDD0FCB4479), - ULL(0xE84F826623588071), - ULL(0x0D0CA5F146CAF8F0), - ULL(0xC78AEADF8DEE09F2), - ULL(0x22C9CD48E87C7173), - ULL(0xB7C553157E359377), - ULL(0x528674821BA7EBF6), - ULL(0x98003BACD0831AF4), - ULL(0x7D431C3BB5116275), - ULL(0xCC3242DA89A492E4), - ULL(0x2971654DEC36EA65), - ULL(0xE3F72A6327121B67), - ULL(0x06B40DF4428063E6), - ULL(0x93B893A9D4C981E2), - ULL(0x76FBB43EB15BF963), - ULL(0xBC7DFB107A7F0861), - ULL(0x593EDC871FED70E0), - ULL(0x7327E03C337EB4E8), - ULL(0x9664C7AB56ECCC69), - ULL(0x5CE288859DC83D6B), - ULL(0xB9A1AF12F85A45EA), - ULL(0x2CAD314F6E13A7EE), - ULL(0xC9EE16D80B81DF6F), - ULL(0x036859F6C0A52E6D), - ULL(0xE62B7E61A53756EC), - ULL(0xC6AD22723272EC29), - ULL(0x23EE05E557E094A8), - ULL(0xE9684ACB9CC465AA), - ULL(0x0C2B6D5CF9561D2B), - ULL(0x9927F3016F1FFF2F), - ULL(0x7C64D4960A8D87AE), - ULL(0xB6E29BB8C1A976AC), - ULL(0x53A1BC2FA43B0E2D), - ULL(0x79B8809488A8CA25), - ULL(0x9CFBA703ED3AB2A4), - ULL(0x567DE82D261E43A6), - ULL(0xB33ECFBA438C3B27), - ULL(0x263251E7D5C5D923), - ULL(0xC3717670B057A1A2), - ULL(0x09F7395E7B7350A0), - ULL(0xECB41EC91EE12821), - ULL(0x5DC540282254D8B0), - ULL(0xB88667BF47C6A031), - ULL(0x720028918CE25133), - ULL(0x97430F06E97029B2), - ULL(0x024F915B7F39CBB6), - ULL(0xE70CB6CC1AABB337), - ULL(0x2D8AF9E2D18F4235), - ULL(0xC8C9DE75B41D3AB4), - ULL(0xE2D0E2CE988EFEBC), - ULL(0x0793C559FD1C863D), - ULL(0xCD158A773638773F), - ULL(0x2856ADE053AA0FBE), - ULL(0xBD5A33BDC5E3EDBA), - ULL(0x5819142AA071953B), - ULL(0x929F5B046B556439), - ULL(0x77DC7C930EC71CB8), - ULL(0x153EC15177ACFD9A), - ULL(0xF07DE6C6123E851B), - ULL(0x3AFBA9E8D91A7419), - ULL(0xDFB88E7FBC880C98), - ULL(0x4AB410222AC1EE9C), - ULL(0xAFF737B54F53961D), - ULL(0x6571789B8477671F), - ULL(0x80325F0CE1E51F9E), - ULL(0xAA2B63B7CD76DB96), - ULL(0x4F684420A8E4A317), - ULL(0x85EE0B0E63C05215), - ULL(0x60AD2C9906522A94), - ULL(0xF5A1B2C4901BC890), - ULL(0x10E29553F589B011), - ULL(0xDA64DA7D3EAD4113), - ULL(0x3F27FDEA5B3F3992), - ULL(0x8E56A30B678AC903), - ULL(0x6B15849C0218B182), - ULL(0xA193CBB2C93C4080), - ULL(0x44D0EC25ACAE3801), - ULL(0xD1DC72783AE7DA05), - ULL(0x349F55EF5F75A284), - ULL(0xFE191AC194515386), - ULL(0x1B5A3D56F1C32B07), - ULL(0x314301EDDD50EF0F), - ULL(0xD400267AB8C2978E), - ULL(0x1E86695473E6668C), - ULL(0xFBC54EC316741E0D), - ULL(0x6EC9D09E803DFC09), - ULL(0x8B8AF709E5AF8488), - ULL(0x410CB8272E8B758A), - ULL(0xA44F9FB04B190D0B), + ULL(0x0000000000000000), + ULL(0xE543279765927881), + ULL(0x2FC568B9AEB68983), + ULL(0xCA864F2ECB24F102), + ULL(0x5F8AD1735D6D1306), + ULL(0xBAC9F6E438FF6B87), + ULL(0x704FB9CAF3DB9A85), + ULL(0x950C9E5D9649E204), + ULL(0xBF15A2E6BADA260C), + ULL(0x5A568571DF485E8D), + ULL(0x90D0CA5F146CAF8F), + ULL(0x7593EDC871FED70E), + ULL(0xE09F7395E7B7350A), + ULL(0x05DC540282254D8B), + ULL(0xCF5A1B2C4901BC89), + ULL(0x2A193CBB2C93C408), + ULL(0x9B68625A10263499), + ULL(0x7E2B45CD75B44C18), + ULL(0xB4AD0AE3BE90BD1A), + ULL(0x51EE2D74DB02C59B), + ULL(0xC4E2B3294D4B279F), + ULL(0x21A194BE28D95F1E), + ULL(0xEB27DB90E3FDAE1C), + ULL(0x0E64FC07866FD69D), + ULL(0x247DC0BCAAFC1295), + ULL(0xC13EE72BCF6E6A14), + ULL(0x0BB8A805044A9B16), + ULL(0xEEFB8F9261D8E397), + ULL(0x7BF711CFF7910193), + ULL(0x9EB4365892037912), + ULL(0x5432797659278810), + ULL(0xB1715EE13CB5F091), + ULL(0xD393E32345DE11B3), + ULL(0x36D0C4B4204C6932), + ULL(0xFC568B9AEB689830), + ULL(0x1915AC0D8EFAE0B1), + ULL(0x8C19325018B302B5), + ULL(0x695A15C77D217A34), + ULL(0xA3DC5AE9B6058B36), + ULL(0x469F7D7ED397F3B7), + ULL(0x6C8641C5FF0437BF), + ULL(0x89C566529A964F3E), + ULL(0x4343297C51B2BE3C), + ULL(0xA6000EEB3420C6BD), + ULL(0x330C90B6A26924B9), + ULL(0xD64FB721C7FB5C38), + ULL(0x1CC9F80F0CDFAD3A), + ULL(0xF98ADF98694DD5BB), + ULL(0x48FB817955F8252A), + ULL(0xADB8A6EE306A5DAB), + ULL(0x673EE9C0FB4EACA9), + ULL(0x827DCE579EDCD428), + ULL(0x1771500A0895362C), + ULL(0xF232779D6D074EAD), + ULL(0x38B438B3A623BFAF), + ULL(0xDDF71F24C3B1C72E), + ULL(0xF7EE239FEF220326), + ULL(0x12AD04088AB07BA7), + ULL(0xD82B4B2641948AA5), + ULL(0x3D686CB12406F224), + ULL(0xA864F2ECB24F1020), + ULL(0x4D27D57BD7DD68A1), + ULL(0x87A19A551CF999A3), + ULL(0x62E2BDC2796BE122), + ULL(0x4264E1D1EE2E5BE7), + ULL(0xA727C6468BBC2366), + ULL(0x6DA189684098D264), + ULL(0x88E2AEFF250AAAE5), + ULL(0x1DEE30A2B34348E1), + ULL(0xF8AD1735D6D13060), + ULL(0x322B581B1DF5C162), + ULL(0xD7687F8C7867B9E3), + ULL(0xFD71433754F47DEB), + ULL(0x183264A03166056A), + ULL(0xD2B42B8EFA42F468), + ULL(0x37F70C199FD08CE9), + ULL(0xA2FB924409996EED), + ULL(0x47B8B5D36C0B166C), + ULL(0x8D3EFAFDA72FE76E), + ULL(0x687DDD6AC2BD9FEF), + ULL(0xD90C838BFE086F7E), + ULL(0x3C4FA41C9B9A17FF), + ULL(0xF6C9EB3250BEE6FD), + ULL(0x138ACCA5352C9E7C), + ULL(0x868652F8A3657C78), + ULL(0x63C5756FC6F704F9), + ULL(0xA9433A410DD3F5FB), + ULL(0x4C001DD668418D7A), + ULL(0x6619216D44D24972), + ULL(0x835A06FA214031F3), + ULL(0x49DC49D4EA64C0F1), + ULL(0xAC9F6E438FF6B870), + ULL(0x3993F01E19BF5A74), + ULL(0xDCD0D7897C2D22F5), + ULL(0x165698A7B709D3F7), + ULL(0xF315BF30D29BAB76), + ULL(0x91F702F2ABF04A54), + ULL(0x74B42565CE6232D5), + ULL(0xBE326A4B0546C3D7), + ULL(0x5B714DDC60D4BB56), + ULL(0xCE7DD381F69D5952), + ULL(0x2B3EF416930F21D3), + ULL(0xE1B8BB38582BD0D1), + ULL(0x04FB9CAF3DB9A850), + ULL(0x2EE2A014112A6C58), + ULL(0xCBA1878374B814D9), + ULL(0x0127C8ADBF9CE5DB), + ULL(0xE464EF3ADA0E9D5A), + ULL(0x716871674C477F5E), + ULL(0x942B56F029D507DF), + ULL(0x5EAD19DEE2F1F6DD), + ULL(0xBBEE3E4987638E5C), + ULL(0x0A9F60A8BBD67ECD), + ULL(0xEFDC473FDE44064C), + ULL(0x255A08111560F74E), + ULL(0xC0192F8670F28FCF), + ULL(0x5515B1DBE6BB6DCB), + ULL(0xB056964C8329154A), + ULL(0x7AD0D962480DE448), + ULL(0x9F93FEF52D9F9CC9), + ULL(0xB58AC24E010C58C1), + ULL(0x50C9E5D9649E2040), + ULL(0x9A4FAAF7AFBAD142), + ULL(0x7F0C8D60CA28A9C3), + ULL(0xEA00133D5C614BC7), + ULL(0x0F4334AA39F33346), + ULL(0xC5C57B84F2D7C244), + ULL(0x20865C139745BAC5), + ULL(0x84C9C3A3DC5CB7CE), + ULL(0x618AE434B9CECF4F), + ULL(0xAB0CAB1A72EA3E4D), + ULL(0x4E4F8C8D177846CC), + ULL(0xDB4312D08131A4C8), + ULL(0x3E003547E4A3DC49), + ULL(0xF4867A692F872D4B), + ULL(0x11C55DFE4A1555CA), + ULL(0x3BDC6145668691C2), + ULL(0xDE9F46D20314E943), + ULL(0x141909FCC8301841), + ULL(0xF15A2E6BADA260C0), + ULL(0x6456B0363BEB82C4), + ULL(0x811597A15E79FA45), + ULL(0x4B93D88F955D0B47), + ULL(0xAED0FF18F0CF73C6), + ULL(0x1FA1A1F9CC7A8357), + ULL(0xFAE2866EA9E8FBD6), + ULL(0x3064C94062CC0AD4), + ULL(0xD527EED7075E7255), + ULL(0x402B708A91179051), + ULL(0xA568571DF485E8D0), + ULL(0x6FEE18333FA119D2), + ULL(0x8AAD3FA45A336153), + ULL(0xA0B4031F76A0A55B), + ULL(0x45F724881332DDDA), + ULL(0x8F716BA6D8162CD8), + ULL(0x6A324C31BD845459), + ULL(0xFF3ED26C2BCDB65D), + ULL(0x1A7DF5FB4E5FCEDC), + ULL(0xD0FBBAD5857B3FDE), + ULL(0x35B89D42E0E9475F), + ULL(0x575A20809982A67D), + ULL(0xB2190717FC10DEFC), + ULL(0x789F483937342FFE), + ULL(0x9DDC6FAE52A6577F), + ULL(0x08D0F1F3C4EFB57B), + ULL(0xED93D664A17DCDFA), + ULL(0x2715994A6A593CF8), + ULL(0xC256BEDD0FCB4479), + ULL(0xE84F826623588071), + ULL(0x0D0CA5F146CAF8F0), + ULL(0xC78AEADF8DEE09F2), + ULL(0x22C9CD48E87C7173), + ULL(0xB7C553157E359377), + ULL(0x528674821BA7EBF6), + ULL(0x98003BACD0831AF4), + ULL(0x7D431C3BB5116275), + ULL(0xCC3242DA89A492E4), + ULL(0x2971654DEC36EA65), + ULL(0xE3F72A6327121B67), + ULL(0x06B40DF4428063E6), + ULL(0x93B893A9D4C981E2), + ULL(0x76FBB43EB15BF963), + ULL(0xBC7DFB107A7F0861), + ULL(0x593EDC871FED70E0), + ULL(0x7327E03C337EB4E8), + ULL(0x9664C7AB56ECCC69), + ULL(0x5CE288859DC83D6B), + ULL(0xB9A1AF12F85A45EA), + ULL(0x2CAD314F6E13A7EE), + ULL(0xC9EE16D80B81DF6F), + ULL(0x036859F6C0A52E6D), + ULL(0xE62B7E61A53756EC), + ULL(0xC6AD22723272EC29), + ULL(0x23EE05E557E094A8), + ULL(0xE9684ACB9CC465AA), + ULL(0x0C2B6D5CF9561D2B), + ULL(0x9927F3016F1FFF2F), + ULL(0x7C64D4960A8D87AE), + ULL(0xB6E29BB8C1A976AC), + ULL(0x53A1BC2FA43B0E2D), + ULL(0x79B8809488A8CA25), + ULL(0x9CFBA703ED3AB2A4), + ULL(0x567DE82D261E43A6), + ULL(0xB33ECFBA438C3B27), + ULL(0x263251E7D5C5D923), + ULL(0xC3717670B057A1A2), + ULL(0x09F7395E7B7350A0), + ULL(0xECB41EC91EE12821), + ULL(0x5DC540282254D8B0), + ULL(0xB88667BF47C6A031), + ULL(0x720028918CE25133), + ULL(0x97430F06E97029B2), + ULL(0x024F915B7F39CBB6), + ULL(0xE70CB6CC1AABB337), + ULL(0x2D8AF9E2D18F4235), + ULL(0xC8C9DE75B41D3AB4), + ULL(0xE2D0E2CE988EFEBC), + ULL(0x0793C559FD1C863D), + ULL(0xCD158A773638773F), + ULL(0x2856ADE053AA0FBE), + ULL(0xBD5A33BDC5E3EDBA), + ULL(0x5819142AA071953B), + ULL(0x929F5B046B556439), + ULL(0x77DC7C930EC71CB8), + ULL(0x153EC15177ACFD9A), + ULL(0xF07DE6C6123E851B), + ULL(0x3AFBA9E8D91A7419), + ULL(0xDFB88E7FBC880C98), + ULL(0x4AB410222AC1EE9C), + ULL(0xAFF737B54F53961D), + ULL(0x6571789B8477671F), + ULL(0x80325F0CE1E51F9E), + ULL(0xAA2B63B7CD76DB96), + ULL(0x4F684420A8E4A317), + ULL(0x85EE0B0E63C05215), + ULL(0x60AD2C9906522A94), + ULL(0xF5A1B2C4901BC890), + ULL(0x10E29553F589B011), + ULL(0xDA64DA7D3EAD4113), + ULL(0x3F27FDEA5B3F3992), + ULL(0x8E56A30B678AC903), + ULL(0x6B15849C0218B182), + ULL(0xA193CBB2C93C4080), + ULL(0x44D0EC25ACAE3801), + ULL(0xD1DC72783AE7DA05), + ULL(0x349F55EF5F75A284), + ULL(0xFE191AC194515386), + ULL(0x1B5A3D56F1C32B07), + ULL(0x314301EDDD50EF0F), + ULL(0xD400267AB8C2978E), + ULL(0x1E86695473E6668C), + ULL(0xFBC54EC316741E0D), + ULL(0x6EC9D09E803DFC09), + ULL(0x8B8AF709E5AF8488), + ULL(0x410CB8272E8B758A), + ULL(0xA44F9FB04B190D0B), }; -const ui32* crctab16 = CRCTAB16; -const ui32* crctab32 = CRCTAB32; -const ui64* crctab64 = CRCTAB64; +const ui32* crctab16 = CRCTAB16; +const ui32* crctab32 = CRCTAB32; +const ui64* crctab64 = CRCTAB64; diff --git a/library/cpp/digest/old_crc/crc.h b/library/cpp/digest/old_crc/crc.h index 4a3ce6d05e..e7da8967a8 100644 --- a/library/cpp/digest/old_crc/crc.h +++ b/library/cpp/digest/old_crc/crc.h @@ -7,19 +7,19 @@ #define CRC64INIT ULL(0xFFFFFFFFFFFFFFFF) // CCITT CRC-16 -inline ui16 crc16(const char* buf, size_t buflen, ui32 crcinit = CRC16INIT) { +inline ui16 crc16(const char* buf, size_t buflen, ui32 crcinit = CRC16INIT) { ui32 crc = 0xFFFF & ((crcinit >> 8) ^ (crcinit << 8)); - const char* end = buf + buflen; - extern const ui32* crctab16; + const char* end = buf + buflen; + extern const ui32* crctab16; while (buf < end) { crc = (crc >> 8) ^ crctab16[(crc ^ *buf) & 0xFF]; ++buf; } - return (ui16)(0xFFFF & ((crc >> 8) ^ (crc << 8))); + return (ui16)(0xFFFF & ((crc >> 8) ^ (crc << 8))); } -struct IdTR { +struct IdTR { Y_FORCE_INLINE static ui8 T(ui8 a) { return a; } @@ -27,31 +27,31 @@ struct IdTR { // CCITT CRC-32 template <class TR> -inline ui32 crc32(const char* buf, size_t buflen, ui32 crcinit = CRC32INIT) { +inline ui32 crc32(const char* buf, size_t buflen, ui32 crcinit = CRC32INIT) { ui32 crc = crcinit ^ 0xFFFFFFFF; - const char* end = buf + buflen; - extern const ui32* crctab32; + const char* end = buf + buflen; + extern const ui32* crctab32; - while (buf < end) { + while (buf < end) { crc = (crc >> 8) ^ crctab32[(crc ^ TR::T((ui8)*buf)) & 0xFF]; ++buf; } - + return crc ^ 0xFFFFFFFF; } -inline ui32 crc32(const char* buf, size_t buflen, ui32 crcinit = CRC32INIT) { +inline ui32 crc32(const char* buf, size_t buflen, ui32 crcinit = CRC32INIT) { return crc32<IdTR>(buf, buflen, crcinit); } -inline ui32 crc32(const void* buf, size_t buflen, ui32 crcinit = CRC32INIT) { - return crc32((const char*)buf, buflen, crcinit); +inline ui32 crc32(const void* buf, size_t buflen, ui32 crcinit = CRC32INIT) { + return crc32((const char*)buf, buflen, crcinit); } // Copyright (C) Sewell Development Corporation, 1994 - 1998. -inline ui64 crc64(const void* buf, size_t buflen, ui64 crcinit = CRC64INIT) { - const unsigned char* ptr = (const unsigned char*)buf; - extern const ui64* crctab64; +inline ui64 crc64(const void* buf, size_t buflen, ui64 crcinit = CRC64INIT) { + const unsigned char* ptr = (const unsigned char*)buf; + extern const ui64* crctab64; while (buflen--) { crcinit = crctab64[((crcinit >> 56) ^ *ptr++)] ^ (crcinit << 8); @@ -59,32 +59,32 @@ inline ui64 crc64(const void* buf, size_t buflen, ui64 crcinit = CRC64INIT) { return crcinit; } -namespace NCrcPrivate { - template <unsigned N> - struct TCrcHelper; - -#define DEF_CRC_FUNC(t) \ - template <> \ - struct TCrcHelper<t> { \ - static const ui##t Init = CRC##t##INIT; \ - static inline ui##t Crc(const void* buf, size_t buflen, ui##t crcinit) { \ - return crc##t((const char*)buf, buflen, crcinit); \ - } \ - }; - - DEF_CRC_FUNC(16) - DEF_CRC_FUNC(32) - DEF_CRC_FUNC(64) - -#undef DEF_CRC_FUNC -} - +namespace NCrcPrivate { + template <unsigned N> + struct TCrcHelper; + +#define DEF_CRC_FUNC(t) \ + template <> \ + struct TCrcHelper<t> { \ + static const ui##t Init = CRC##t##INIT; \ + static inline ui##t Crc(const void* buf, size_t buflen, ui##t crcinit) { \ + return crc##t((const char*)buf, buflen, crcinit); \ + } \ + }; + + DEF_CRC_FUNC(16) + DEF_CRC_FUNC(32) + DEF_CRC_FUNC(64) + +#undef DEF_CRC_FUNC +} + template <class T> -static inline T Crc(const void* buf, size_t len, T init) { - return (T)NCrcPrivate::TCrcHelper<8 * sizeof(T)>::Crc(buf, len, init); +static inline T Crc(const void* buf, size_t len, T init) { + return (T)NCrcPrivate::TCrcHelper<8 * sizeof(T)>::Crc(buf, len, init); } -template <class T> -static inline T Crc(const void* buf, size_t len) { - return Crc<T>(buf, len, (T)NCrcPrivate::TCrcHelper<8 * sizeof(T)>::Init); -} +template <class T> +static inline T Crc(const void* buf, size_t len) { + return Crc<T>(buf, len, (T)NCrcPrivate::TCrcHelper<8 * sizeof(T)>::Init); +} diff --git a/library/cpp/digest/old_crc/crc_ut.cpp b/library/cpp/digest/old_crc/crc_ut.cpp index 46e1d5d29b..6bf2a12294 100644 --- a/library/cpp/digest/old_crc/crc_ut.cpp +++ b/library/cpp/digest/old_crc/crc_ut.cpp @@ -1,74 +1,74 @@ -#include "crc.h" - +#include "crc.h" + #include <library/cpp/testing/unittest/registar.h> - + #include <util/stream/output.h> - -class TCrcTest: public TTestBase { - UNIT_TEST_SUITE(TCrcTest); - UNIT_TEST(TestCrc16) - UNIT_TEST(TestCrc32) - UNIT_TEST(TestCrc64) - UNIT_TEST_SUITE_END(); - -private: - inline void TestCrc16() { - const ui16 expected[] = { - 41719, - 57001, - 53722, - 3276}; - - Run<ui16>(expected); - } - - inline void TestCrc32() { - const ui32 expected[] = { - 688229491UL, - 3543112608UL, - 4127534015UL, - 3009909774UL}; - - Run<ui32>(expected); - } - - inline void TestCrc64() { - const ui64 expected[] = { - 12116107829328640258ULL, - 18186277744016380552ULL, - 249923753044811734ULL, - 7852471725963920356ULL}; - - Run<ui64>(expected); - } - -private: - template <class T> - inline void Run(const T* expected) { - ui8 buf[256]; - - for (size_t i = 0; i < 256; ++i) { - buf[i] = i; - } - - Test<T>(buf, 256, expected[0]); - Test<T>(buf, 255, expected[1]); - Test<T>(buf, 254, expected[2]); - Test<T>(buf, 253, expected[3]); - } - - template <class T> - inline void Test(const void* data, size_t len, T expected) { - const T res = Crc<T>(data, len); - - try { - UNIT_ASSERT_EQUAL(res, expected); - } catch (...) { - Cerr << res << Endl; - - throw; - } - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TCrcTest); + +class TCrcTest: public TTestBase { + UNIT_TEST_SUITE(TCrcTest); + UNIT_TEST(TestCrc16) + UNIT_TEST(TestCrc32) + UNIT_TEST(TestCrc64) + UNIT_TEST_SUITE_END(); + +private: + inline void TestCrc16() { + const ui16 expected[] = { + 41719, + 57001, + 53722, + 3276}; + + Run<ui16>(expected); + } + + inline void TestCrc32() { + const ui32 expected[] = { + 688229491UL, + 3543112608UL, + 4127534015UL, + 3009909774UL}; + + Run<ui32>(expected); + } + + inline void TestCrc64() { + const ui64 expected[] = { + 12116107829328640258ULL, + 18186277744016380552ULL, + 249923753044811734ULL, + 7852471725963920356ULL}; + + Run<ui64>(expected); + } + +private: + template <class T> + inline void Run(const T* expected) { + ui8 buf[256]; + + for (size_t i = 0; i < 256; ++i) { + buf[i] = i; + } + + Test<T>(buf, 256, expected[0]); + Test<T>(buf, 255, expected[1]); + Test<T>(buf, 254, expected[2]); + Test<T>(buf, 253, expected[3]); + } + + template <class T> + inline void Test(const void* data, size_t len, T expected) { + const T res = Crc<T>(data, len); + + try { + UNIT_ASSERT_EQUAL(res, expected); + } catch (...) { + Cerr << res << Endl; + + throw; + } + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TCrcTest); diff --git a/library/cpp/digest/old_crc/gencrc/main.cpp b/library/cpp/digest/old_crc/gencrc/main.cpp index d5821304ce..e3fe9e04ce 100644 --- a/library/cpp/digest/old_crc/gencrc/main.cpp +++ b/library/cpp/digest/old_crc/gencrc/main.cpp @@ -4,29 +4,29 @@ #define POLY_32 0xEDB88320UL #define POLY_64 ULL(0xE543279765927881) -static void crc16init() { - ui32 CRCTAB16[256]; +static void crc16init() { + ui32 CRCTAB16[256]; ui32 crc; int i, j; - for (i = 0; i < 256; CRCTAB16[i++] = 0xFFFF & ((crc << 8) ^ (crc >> 8))) + for (i = 0; i < 256; CRCTAB16[i++] = 0xFFFF & ((crc << 8) ^ (crc >> 8))) for (crc = i, j = 8; j > 0; j--) if (crc & 1) crc = (crc >> 1) ^ POLY_16; else crc >>= 1; - + for (size_t k = 0; k < 256; ++k) { Cout << " ULL(" << CRCTAB16[k] << ")"; - + if (k != 255) { - Cout << ",\n"; - } - } + Cout << ",\n"; + } + } } -static void crc32init() { - ui32 CRCTAB32[256]; +static void crc32init() { + ui32 CRCTAB32[256]; ui32 crc; int i, j; @@ -36,21 +36,21 @@ static void crc32init() { crc = (crc >> 1) ^ POLY_32; else crc >>= 1; - + for (size_t k = 0; k < 256; ++k) { Cout << " ULL(" << CRCTAB32[k] << ")"; - + if (k != 255) { - Cout << ",\n"; - } - } + Cout << ",\n"; + } + } } -int main() { - Cout << "static const ui32 CRCTAB16[] = {\n\n"; - crc16init(); - Cout << "\n};\n\n"; - Cout << "static const ui32 CRCTAB32[] = {\n\n"; - crc32init(); - Cout << "\n};\n\n"; -} +int main() { + Cout << "static const ui32 CRCTAB16[] = {\n\n"; + crc16init(); + Cout << "\n};\n\n"; + Cout << "static const ui32 CRCTAB32[] = {\n\n"; + crc32init(); + Cout << "\n};\n\n"; +} diff --git a/library/cpp/digest/old_crc/gencrc/ya.make b/library/cpp/digest/old_crc/gencrc/ya.make index df25863bee..4d3a328780 100644 --- a/library/cpp/digest/old_crc/gencrc/ya.make +++ b/library/cpp/digest/old_crc/gencrc/ya.make @@ -1,9 +1,9 @@ -PROGRAM() - -OWNER(pg) - -SRCS( - main.cpp -) - -END() +PROGRAM() + +OWNER(pg) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/digest/old_crc/ut/ya.make b/library/cpp/digest/old_crc/ut/ya.make index a783bbae5b..08e432b351 100644 --- a/library/cpp/digest/old_crc/ut/ya.make +++ b/library/cpp/digest/old_crc/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/digest/old_crc) - + OWNER( pg g:util ) - -SRCS( - crc_ut.cpp -) - -END() + +SRCS( + crc_ut.cpp +) + +END() diff --git a/library/cpp/digest/old_crc/ya.make b/library/cpp/digest/old_crc/ya.make index aa6ea8f6c5..66eaa0f4ad 100644 --- a/library/cpp/digest/old_crc/ya.make +++ b/library/cpp/digest/old_crc/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - +LIBRARY() + OWNER( pg g:util ) - -SRCS( - crc.cpp -) - + +SRCS( + crc.cpp +) + RUN_PROGRAM( library/cpp/digest/old_crc/gencrc STDOUT crc.inc ) - -END() + +END() diff --git a/library/cpp/digest/sfh/sfh.cpp b/library/cpp/digest/sfh/sfh.cpp index c10062105b..c36724a3f1 100644 --- a/library/cpp/digest/sfh/sfh.cpp +++ b/library/cpp/digest/sfh/sfh.cpp @@ -1 +1 @@ -#include "sfh.h" +#include "sfh.h" diff --git a/library/cpp/digest/sfh/sfh.h b/library/cpp/digest/sfh/sfh.h index 372938654c..cce9558916 100644 --- a/library/cpp/digest/sfh/sfh.h +++ b/library/cpp/digest/sfh/sfh.h @@ -4,37 +4,37 @@ #include <util/system/unaligned_mem.h> inline ui32 SuperFastHash(const void* d, size_t l) noexcept { - ui32 hash = (ui32)l; - ui32 tmp; + ui32 hash = (ui32)l; + ui32 tmp; - if (!l || !d) + if (!l || !d) return 0; - TUnalignedMemoryIterator<ui16, 4> iter(d, l); + TUnalignedMemoryIterator<ui16, 4> iter(d, l); - while (!iter.AtEnd()) { - hash += (ui32)iter.Next(); - tmp = ((ui32)iter.Next() << 11) ^ hash; - hash = (hash << 16) ^ tmp; - hash += hash >> 11; + while (!iter.AtEnd()) { + hash += (ui32)iter.Next(); + tmp = ((ui32)iter.Next() << 11) ^ hash; + hash = (hash << 16) ^ tmp; + hash += hash >> 11; } - switch (iter.Left()) { - case 3: - hash += (ui32)iter.Next(); + switch (iter.Left()) { + case 3: + hash += (ui32)iter.Next(); hash ^= hash << 16; - hash ^= ((ui32)(i32) * (const i8*)iter.Last()) << 18; + hash ^= ((ui32)(i32) * (const i8*)iter.Last()) << 18; hash += hash >> 11; break; - - case 2: - hash += (ui32)iter.Cur(); + + case 2: + hash += (ui32)iter.Cur(); hash ^= hash << 11; hash += hash >> 17; break; - - case 1: - hash += *((const i8*)iter.Last()); + + case 1: + hash += *((const i8*)iter.Last()); hash ^= hash << 10; hash += hash >> 1; } diff --git a/library/cpp/digest/sfh/sfh_ut.cpp b/library/cpp/digest/sfh/sfh_ut.cpp index 912999bae7..79606232de 100644 --- a/library/cpp/digest/sfh/sfh_ut.cpp +++ b/library/cpp/digest/sfh/sfh_ut.cpp @@ -1,40 +1,40 @@ -#include "sfh.h" - +#include "sfh.h" + #include <library/cpp/testing/unittest/registar.h> - + #include <util/stream/output.h> - -class TSfhTest: public TTestBase { - UNIT_TEST_SUITE(TSfhTest); - UNIT_TEST(TestSfh) - UNIT_TEST_SUITE_END(); - -private: - inline void TestSfh() { - ui8 buf[256]; - - for (size_t i = 0; i < 256; ++i) { - buf[i] = i; - } - - Test(buf, 256, 3840866583UL); - Test(buf, 255, 325350515UL); - Test(buf, 254, 2920741773UL); - Test(buf, 253, 3586628615UL); - } - -private: - inline void Test(const void* data, size_t len, ui32 expected) { - const ui32 res = SuperFastHash(data, len); - - try { - UNIT_ASSERT_EQUAL(res, expected); - } catch (...) { - Cerr << res << ", " << expected << Endl; - - throw; - } - } -}; - -UNIT_TEST_SUITE_REGISTRATION(TSfhTest); + +class TSfhTest: public TTestBase { + UNIT_TEST_SUITE(TSfhTest); + UNIT_TEST(TestSfh) + UNIT_TEST_SUITE_END(); + +private: + inline void TestSfh() { + ui8 buf[256]; + + for (size_t i = 0; i < 256; ++i) { + buf[i] = i; + } + + Test(buf, 256, 3840866583UL); + Test(buf, 255, 325350515UL); + Test(buf, 254, 2920741773UL); + Test(buf, 253, 3586628615UL); + } + +private: + inline void Test(const void* data, size_t len, ui32 expected) { + const ui32 res = SuperFastHash(data, len); + + try { + UNIT_ASSERT_EQUAL(res, expected); + } catch (...) { + Cerr << res << ", " << expected << Endl; + + throw; + } + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TSfhTest); diff --git a/library/cpp/digest/sfh/ut/ya.make b/library/cpp/digest/sfh/ut/ya.make index 256a66295a..73639cb339 100644 --- a/library/cpp/digest/sfh/ut/ya.make +++ b/library/cpp/digest/sfh/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/digest/sfh) - + OWNER( pg g:util ) - -SRCS( - sfh_ut.cpp -) - -END() + +SRCS( + sfh_ut.cpp +) + +END() diff --git a/library/cpp/digest/sfh/ya.make b/library/cpp/digest/sfh/ya.make index b906cd1577..0424a5f3b8 100644 --- a/library/cpp/digest/sfh/ya.make +++ b/library/cpp/digest/sfh/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - +LIBRARY() + OWNER( pg g:util ) - -SRCS( - sfh.cpp -) - -END() + +SRCS( + sfh.cpp +) + +END() diff --git a/library/cpp/digest/ya.make b/library/cpp/digest/ya.make index 61d4b50903..e378bb1033 100644 --- a/library/cpp/digest/ya.make +++ b/library/cpp/digest/ya.make @@ -3,21 +3,21 @@ RECURSE( argonish/benchmark argonish/ut_fat argonish/ut - benchmark - fast - lower_case - lower_case/ut - md5 - md5/ut + benchmark + fast + lower_case + lower_case/ut + md5 + md5/ut md5/medium_ut - md5/bench + md5/bench murmur murmur/ut - crc32c - crc32c/ut - old_crc - old_crc/ut - old_crc/gencrc - sfh - sfh/ut + crc32c + crc32c/ut + old_crc + old_crc/ut + old_crc/gencrc + sfh + sfh/ut ) diff --git a/library/cpp/dns/cache.cpp b/library/cpp/dns/cache.cpp index 05c14e82fc..6d2dbe9ec9 100644 --- a/library/cpp/dns/cache.cpp +++ b/library/cpp/dns/cache.cpp @@ -1,121 +1,121 @@ #include "cache.h" - + #include "thread.h" #include <util/system/tls.h> #include <util/system/info.h> #include <util/system/rwlock.h> -#include <util/thread/singleton.h> +#include <util/thread/singleton.h> #include <util/generic/singleton.h> -#include <util/generic/hash.h> - +#include <util/generic/hash.h> + using namespace NDns; - -namespace { + +namespace { struct TResolveTask { enum EMethod { - Normal, - Threaded + Normal, + Threaded }; - + inline TResolveTask(const TResolveInfo& info, EMethod method) : Info(info) , Method(method) { } - + const TResolveInfo& Info; const EMethod Method; }; - + class IDns { public: virtual ~IDns() = default; virtual const TResolvedHost* Resolve(const TResolveTask&) = 0; }; - typedef TAtomicSharedPtr<TResolvedHost> TResolvedHostPtr; - + typedef TAtomicSharedPtr<TResolvedHost> TResolvedHostPtr; + struct THashResolveInfo { inline size_t operator()(const TResolveInfo& ri) const { return ComputeHash(ri.Host) ^ ri.Port; } }; - + struct TCompareResolveInfo { inline bool operator()(const NDns::TResolveInfo& x, const NDns::TResolveInfo& y) const { return x.Host == y.Host && x.Port == y.Port; } }; - + class TGlobalCachedDns: public IDns, public TNonCopyable { public: const TResolvedHost* Resolve(const TResolveTask& rt) override { //2. search host in cache { TReadGuard guard(L_); - + TCache::const_iterator it = C_.find(rt.Info); - + if (it != C_.end()) { return it->second.Get(); - } - } - + } + } + TResolvedHostPtr res = ResolveA(rt); - + //update cache { TWriteGuard guard(L_); - + std::pair<TCache::iterator, bool> updateResult = C_.insert(std::make_pair(TResolveInfo(res->Host, rt.Info.Port), res)); - TResolvedHost* rh = updateResult.first->second.Get(); - + TResolvedHost* rh = updateResult.first->second.Get(); + if (updateResult.second) { //fresh resolved host, set cache record id for it rh->Id = C_.size() - 1; } - + return rh; - } + } } - + void AddAlias(const TString& host, const TString& alias) noexcept { TWriteGuard guard(LA_); - + A_[host] = alias; - } - + } + static inline TGlobalCachedDns* Instance() { return SingletonWithPriority<TGlobalCachedDns, 65530>(); - } - - private: + } + + private: inline TResolvedHostPtr ResolveA(const TResolveTask& rt) { TString originalHost(rt.Info.Host); TString host(originalHost); - + //3. replace host to alias, if exist if (A_.size()) { TReadGuard guard(LA_); TStringBuf names[] = {"*", host}; - - for (const auto& name : names) { + + for (const auto& name : names) { TAliases::const_iterator it = A_.find(name); - + if (it != A_.end()) { host = it->second; } - } - } - + } + } + if (host.length() > 2 && host[0] == '[') { TString unbracedIpV6(host.data() + 1, host.size() - 2); host.swap(unbracedIpV6); } TAutoPtr<TNetworkAddress> na; - + //4. getaddrinfo (direct or in separate thread) if (rt.Method == TResolveTask::Normal) { na.Reset(new TNetworkAddress(host, rt.Info.Port)); @@ -125,60 +125,60 @@ namespace { Y_ASSERT(0); throw yexception() << TStringBuf("invalid resolve method"); } - + return new TResolvedHost(originalHost, *na); - } - + } + typedef THashMap<TResolveInfo, TResolvedHostPtr, THashResolveInfo, TCompareResolveInfo> TCache; TCache C_; TRWMutex L_; typedef THashMap<TString, TString> TAliases; TAliases A_; TRWMutex LA_; - }; - - class TCachedDns: public IDns { - public: - inline TCachedDns(IDns* slave) - : S_(slave) - { - } - + }; + + class TCachedDns: public IDns { + public: + inline TCachedDns(IDns* slave) + : S_(slave) + { + } + const TResolvedHost* Resolve(const TResolveTask& rt) override { //1. search in local thread cache - { + { TCache::const_iterator it = C_.find(rt.Info); - - if (it != C_.end()) { - return it->second; - } - } - + + if (it != C_.end()) { + return it->second; + } + } + const TResolvedHost* res = S_->Resolve(rt); - + C_[TResolveInfo(res->Host, rt.Info.Port)] = res; - - return res; - } - - private: + + return res; + } + + private: typedef THashMap<TResolveInfo, const TResolvedHost*, THashResolveInfo, TCompareResolveInfo> TCache; - TCache C_; - IDns* S_; - }; - + TCache C_; + IDns* S_; + }; + struct TThreadedDns: public TCachedDns { - inline TThreadedDns() + inline TThreadedDns() : TCachedDns(TGlobalCachedDns::Instance()) - { - } - }; - + { + } + }; + inline IDns* ThrDns() { - return FastTlsSingleton<TThreadedDns>(); - } -} - + return FastTlsSingleton<TThreadedDns>(); + } +} + namespace NDns { const TResolvedHost* CachedResolve(const TResolveInfo& ri) { TResolveTask rt(ri, TResolveTask::Normal); @@ -195,4 +195,4 @@ namespace NDns { void AddHostAlias(const TString& host, const TString& alias) { TGlobalCachedDns::Instance()->AddAlias(host, alias); } -} +} diff --git a/library/cpp/dns/cache.h b/library/cpp/dns/cache.h index eda5dc4070..a49ffb972f 100644 --- a/library/cpp/dns/cache.h +++ b/library/cpp/dns/cache.h @@ -1,34 +1,34 @@ -#pragma once - -#include <util/network/socket.h> -#include <util/generic/strbuf.h> +#pragma once + +#include <util/network/socket.h> +#include <util/generic/strbuf.h> #include <util/generic/string.h> - + namespace NDns { - struct TResolveInfo { - inline TResolveInfo(const TStringBuf& host, ui16 port) - : Host(host) - , Port(port) - { - } - - TStringBuf Host; - ui16 Port; - }; - - struct TResolvedHost { + struct TResolveInfo { + inline TResolveInfo(const TStringBuf& host, ui16 port) + : Host(host) + , Port(port) + { + } + + TStringBuf Host; + ui16 Port; + }; + + struct TResolvedHost { inline TResolvedHost(const TString& host, const TNetworkAddress& addr) noexcept : Host(host) , Addr(addr) , Id(0) - { - } - + { + } + TString Host; //resolved hostname (from TResolveInfo, - before aliasing) - TNetworkAddress Addr; + TNetworkAddress Addr; size_t Id; //cache record id - }; - + }; + // Resolving order: // 1. check local thread cache, return if found // 2. check global cache, return if found @@ -42,4 +42,4 @@ namespace NDns { //create alias for host, which can be used for static resolving (when alias is ip address) void AddHostAlias(const TString& host, const TString& alias); -} +} diff --git a/library/cpp/dns/magic.cpp b/library/cpp/dns/magic.cpp index b93792146f..239c33fb75 100644 --- a/library/cpp/dns/magic.cpp +++ b/library/cpp/dns/magic.cpp @@ -1,28 +1,28 @@ -#include "magic.h" - -#include <util/generic/yexception.h> - +#include "magic.h" + +#include <util/generic/yexception.h> + using namespace NDns; - -namespace { - namespace NX { - struct TError: public IError { - inline TError() - : E_(std::current_exception()) - { - } - + +namespace { + namespace NX { + struct TError: public IError { + inline TError() + : E_(std::current_exception()) + { + } + void Raise() override { - std::rethrow_exception(E_); - } - - std::exception_ptr E_; - }; - } -} - + std::rethrow_exception(E_); + } + + std::exception_ptr E_; + }; + } +} + IErrorRef NDns::SaveError() { - using namespace NX; - - return new NX::TError(); -} + using namespace NX; + + return new NX::TError(); +} diff --git a/library/cpp/dns/magic.h b/library/cpp/dns/magic.h index d52cde0a6c..94673444cd 100644 --- a/library/cpp/dns/magic.h +++ b/library/cpp/dns/magic.h @@ -1,17 +1,17 @@ -#pragma once - -#include <util/generic/yexception.h> -#include <util/generic/ptr.h> - +#pragma once + +#include <util/generic/yexception.h> +#include <util/generic/ptr.h> + namespace NDns { - class IError { - public: + class IError { + public: virtual ~IError() = default; - - virtual void Raise() = 0; - }; - - typedef TAutoPtr<IError> IErrorRef; - - IErrorRef SaveError(); -} + + virtual void Raise() = 0; + }; + + typedef TAutoPtr<IError> IErrorRef; + + IErrorRef SaveError(); +} diff --git a/library/cpp/dns/thread.cpp b/library/cpp/dns/thread.cpp index 8b27d2d527..ca9c1ff8a1 100644 --- a/library/cpp/dns/thread.cpp +++ b/library/cpp/dns/thread.cpp @@ -1,133 +1,133 @@ #include "thread.h" -#include "magic.h" - +#include "magic.h" + #include <util/network/socket.h> #include <util/thread/factory.h> #include <util/thread/lfqueue.h> #include <util/system/event.h> -#include <util/generic/vector.h> +#include <util/generic/vector.h> #include <util/generic/singleton.h> - + using namespace NDns; - -namespace { + +namespace { class TThreadedResolver: public IThreadFactory::IThreadAble, public TNonCopyable { - struct TResolveRequest { + struct TResolveRequest { inline TResolveRequest(const TString& host, ui16 port) : Host(host) , Port(port) - { - } - + { + } + inline TNetworkAddressPtr Wait() { E.Wait(); - - if (!Error) { - if (!Result) { + + if (!Error) { + if (!Result) { ythrow TNetworkResolutionError(EAI_AGAIN) << TStringBuf(": resolver down"); - } - - return Result; - } - - Error->Raise(); - + } + + return Result; + } + + Error->Raise(); + ythrow TNetworkResolutionError(EAI_FAIL) << TStringBuf(": shit happen"); - } - + } + inline void Resolve() noexcept { - try { + try { Result = new TNetworkAddress(Host, Port); - } catch (...) { - Error = SaveError(); - } - - Wake(); - } - + } catch (...) { + Error = SaveError(); + } + + Wake(); + } + inline void Wake() noexcept { E.Signal(); - } - + } + TString Host; ui16 Port; TManualEvent E; TNetworkAddressPtr Result; - IErrorRef Error; - }; - - public: + IErrorRef Error; + }; + + public: inline TThreadedResolver() : E_(TSystemEvent::rAuto) { T_.push_back(SystemThreadFactory()->Run(this)); - } - + } + inline ~TThreadedResolver() override { Schedule(nullptr); - + for (size_t i = 0; i < T_.size(); ++i) { - T_[i]->Join(); - } - - { + T_[i]->Join(); + } + + { TResolveRequest* rr = nullptr; - - while (Q_.Dequeue(&rr)) { - if (rr) { - rr->Wake(); - } - } - } - } - - static inline TThreadedResolver* Instance() { - return Singleton<TThreadedResolver>(); - } - + + while (Q_.Dequeue(&rr)) { + if (rr) { + rr->Wake(); + } + } + } + } + + static inline TThreadedResolver* Instance() { + return Singleton<TThreadedResolver>(); + } + inline TNetworkAddressPtr Resolve(const TString& host, ui16 port) { TResolveRequest rr(host, port); - - Schedule(&rr); - - return rr.Wait(); - } - - private: - inline void Schedule(TResolveRequest* rr) { - Q_.Enqueue(rr); - E_.Signal(); - } - + + Schedule(&rr); + + return rr.Wait(); + } + + private: + inline void Schedule(TResolveRequest* rr) { + Q_.Enqueue(rr); + E_.Signal(); + } + void DoExecute() override { - while (true) { + while (true) { TResolveRequest* rr = nullptr; - - while (!Q_.Dequeue(&rr)) { - E_.Wait(); - } - - if (rr) { - rr->Resolve(); - } else { - break; - } - } - + + while (!Q_.Dequeue(&rr)) { + E_.Wait(); + } + + if (rr) { + rr->Resolve(); + } else { + break; + } + } + Schedule(nullptr); - } - - private: - TLockFreeQueue<TResolveRequest*> Q_; + } + + private: + TLockFreeQueue<TResolveRequest*> Q_; TSystemEvent E_; typedef TAutoPtr<IThreadFactory::IThread> IThreadRef; TVector<IThreadRef> T_; - }; + }; } - + namespace NDns { TNetworkAddressPtr ThreadedResolve(const TString& host, ui16 port) { return TThreadedResolver::Instance()->Resolve(host, port); - } -} + } +} diff --git a/library/cpp/dns/ut/dns_ut.cpp b/library/cpp/dns/ut/dns_ut.cpp index aae05a742c..5b99ee2a8a 100644 --- a/library/cpp/dns/ut/dns_ut.cpp +++ b/library/cpp/dns/ut/dns_ut.cpp @@ -1,20 +1,20 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/dns/cache.h> #include <util/network/address.h> - + Y_UNIT_TEST_SUITE(TestDNS) { - using namespace NDns; - + using namespace NDns; + Y_UNIT_TEST(TestMagic) { - UNIT_ASSERT_EXCEPTION(CachedThrResolve(TResolveInfo("?", 80)), yexception); - } + UNIT_ASSERT_EXCEPTION(CachedThrResolve(TResolveInfo("?", 80)), yexception); + } Y_UNIT_TEST(TestAsteriskAlias) { AddHostAlias("*", "localhost"); const TResolvedHost* rh = CachedThrResolve(TResolveInfo("yandex.ru", 80)); UNIT_ASSERT(rh != nullptr); - const TNetworkAddress& addr = rh->Addr; + const TNetworkAddress& addr = rh->Addr; for (TNetworkAddress::TIterator ai = addr.Begin(); ai != addr.End(); ai++) { if (ai->ai_family == AF_INET || ai->ai_family == AF_INET6) { NAddr::TAddrInfo info(&*ai); @@ -22,4 +22,4 @@ Y_UNIT_TEST_SUITE(TestDNS) { } } } -} +} diff --git a/library/cpp/dns/ut/ya.make b/library/cpp/dns/ut/ya.make index 7cfd0c4c32..df0c944219 100644 --- a/library/cpp/dns/ut/ya.make +++ b/library/cpp/dns/ut/ya.make @@ -1,16 +1,16 @@ UNITTEST() - + OWNER( and42 pg ) -PEERDIR( +PEERDIR( library/cpp/dns -) - -SRCS( - dns_ut.cpp -) - -END() +) + +SRCS( + dns_ut.cpp +) + +END() diff --git a/library/cpp/enumbitset/enumbitset.h b/library/cpp/enumbitset/enumbitset.h index 41864c3a04..82cb4d34c7 100644 --- a/library/cpp/enumbitset/enumbitset.h +++ b/library/cpp/enumbitset/enumbitset.h @@ -22,20 +22,20 @@ public: typedef TEnumBitSet<TEnum, mbegin, mend> TThis; TEnumBitSet() - : TParent(0) - { - } + : TParent(0) + { + } - explicit TEnumBitSet(const TParent& p) - : TParent(p) - { - } + explicit TEnumBitSet(const TParent& p) + : TParent(p) + { + } void Init(TEnum c) { Set(c); } - template <class... R> + template <class... R> void Init(TEnum c1, TEnum c2, R... r) { Set(c1); Init(c2, r...); @@ -45,7 +45,7 @@ public: Init(c); } - template <class... R> + template <class... R> TEnumBitSet(TEnum c1, TEnum c2, R... r) { Init(c1, c2, r...); } @@ -133,7 +133,7 @@ public: return ret; } - bool operator<(const TThis& right) const { + bool operator<(const TThis& right) const { Y_ASSERT(this->GetChunkCount() == right.GetChunkCount()); for (size_t i = 0; i < this->GetChunkCount(); ++i) { if (this->GetChunks()[i] < right.GetChunks()[i]) @@ -144,48 +144,48 @@ public: return false; } - bool operator!=(const TThis& right) const { - return !(TParent::operator==(right)); + bool operator!=(const TThis& right) const { + return !(TParent::operator==(right)); } - bool operator==(const TThis& right) const { - return TParent::operator==(right); + bool operator==(const TThis& right) const { + return TParent::operator==(right); } - TThis& operator&=(const TThis& right) { - TParent::operator&=(right); + TThis& operator&=(const TThis& right) { + TParent::operator&=(right); return *this; } - TThis& operator|=(const TThis& right) { - TParent::operator|=(right); + TThis& operator|=(const TThis& right) { + TParent::operator|=(right); return *this; } - TThis& operator^=(const TThis& right) { - TParent::operator^=(right); + TThis& operator^=(const TThis& right) { + TParent::operator^=(right); return *this; } - TThis operator~() const { + TThis operator~() const { TThis r = *this; r.Flip(); return r; } - TThis operator|(const TThis& right) const { + TThis operator|(const TThis& right) const { TThis ret = *this; ret |= right; return ret; } - TThis operator&(const TThis& right) const { + TThis operator&(const TThis& right) const { TThis ret = *this; ret &= right; return ret; } - TThis operator^(const TThis& right) const { + TThis operator^(const TThis& right) const { TThis ret = *this; ret ^= right; return ret; @@ -224,7 +224,7 @@ public: return TParent::operator[](this->Pos(e)); } - using TParent::Count; + using TParent::Count; using TParent::Empty; explicit operator bool() const { @@ -243,7 +243,7 @@ public: return TParent::HasAny(mask); } - template <class... R> + template <class... R> bool HasAny(TEnum c1, R... r) const { return Test(c1) || HasAny(r...); } @@ -252,7 +252,7 @@ public: return TParent::HasAll(mask); } - template <class... R> + template <class... R> bool HasAll(TEnum c1, R... r) const { return Test(c1) && HasAll(r...); } @@ -265,7 +265,7 @@ public: } } - void Load(IInputStream* buffer) { + void Load(IInputStream* buffer) { Reset(); ui32 sz = 0; @@ -336,15 +336,15 @@ public: return true; } - bool any() const { // obsolete + bool any() const { // obsolete return !Empty(); } - bool none() const { // obsolete + bool none() const { // obsolete return Empty(); } - size_t count() const { // obsolete + size_t count() const { // obsolete return Count(); } @@ -362,12 +362,12 @@ public: { } - TEnum operator*() const noexcept { + TEnum operator*() const noexcept { Y_ASSERT(Value < EndIndex); return static_cast<TEnum>(Value); } - bool operator!=(const TIterator& other) const noexcept { + bool operator!=(const TIterator& other) const noexcept { return Value != other.Value; } @@ -400,7 +400,7 @@ public: private: static size_t Pos(TEnum c) { Y_ASSERT(IsValid(c)); - return static_cast<size_t>(int(c) - BeginIndex); + return static_cast<size_t>(int(c) - BeginIndex); } bool HasAny(TEnum c) const { @@ -433,19 +433,19 @@ private: }; template <typename TEnum, TEnum mbegin, int mend> -class TSfEnumBitSet: public TEnumBitSet<TEnum, mbegin, mend> { +class TSfEnumBitSet: public TEnumBitSet<TEnum, mbegin, mend> { public: typedef TEnumBitSet<TEnum, mbegin, mend> TParent; TSfEnumBitSet() - : TParent() - { - } + : TParent() + { + } - TSfEnumBitSet(const TParent& p) - : TParent(p) - { - } + TSfEnumBitSet(const TParent& p) + : TParent(p) + { + } //! unsafe initialization from ui64, value must be shifted according to TParent::Begin explicit TSfEnumBitSet(ui64 val) @@ -458,7 +458,7 @@ public: this->SafeSet(c); } - template <class... R> + template <class... R> void Init(TEnum c1, TEnum c2, R... r) { this->SafeSet(c1); Init(c2, r...); @@ -468,7 +468,7 @@ public: Init(c); } - template <class... R> + template <class... R> TSfEnumBitSet(TEnum c1, TEnum c2, R... r) { Init(c1, c2, r...); } diff --git a/library/cpp/enumbitset/enumbitset_ut.cpp b/library/cpp/enumbitset/enumbitset_ut.cpp index e55b3251c3..716c060350 100644 --- a/library/cpp/enumbitset/enumbitset_ut.cpp +++ b/library/cpp/enumbitset/enumbitset_ut.cpp @@ -46,7 +46,7 @@ Y_UNIT_TEST_SUITE(TEnumBitSetTest) { UNIT_ASSERT(!mask.Empty()); UNIT_ASSERT(mask.Count() == 1); if (!mask) - UNIT_ASSERT(false && "should not be empty"); + UNIT_ASSERT(false && "should not be empty"); } Y_UNIT_TEST(TestIter) { diff --git a/library/cpp/execprofile/autostart/start.cpp b/library/cpp/execprofile/autostart/start.cpp index 4fffb03ae1..0fb8c1d594 100644 --- a/library/cpp/execprofile/autostart/start.cpp +++ b/library/cpp/execprofile/autostart/start.cpp @@ -1,15 +1,15 @@ #include <library/cpp/execprofile/profile.h> - -namespace { - struct TInit { - inline TInit() { - BeginProfiling(); - } - - inline ~TInit() { - EndProfiling(); - } - }; - + +namespace { + struct TInit { + inline TInit() { + BeginProfiling(); + } + + inline ~TInit() { + EndProfiling(); + } + }; + const TInit initer; -} +} diff --git a/library/cpp/execprofile/autostart/ya.make b/library/cpp/execprofile/autostart/ya.make index a5b61f075f..72aa5d13fc 100644 --- a/library/cpp/execprofile/autostart/ya.make +++ b/library/cpp/execprofile/autostart/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - -OWNER(pg) - -PEERDIR( +LIBRARY() + +OWNER(pg) + +PEERDIR( library/cpp/execprofile -) - -SRCS( - GLOBAL start.cpp -) - -END() +) + +SRCS( + GLOBAL start.cpp +) + +END() diff --git a/library/cpp/execprofile/profile.cpp b/library/cpp/execprofile/profile.cpp index d05de20203..f13fa0217e 100644 --- a/library/cpp/execprofile/profile.cpp +++ b/library/cpp/execprofile/profile.cpp @@ -2,7 +2,7 @@ #include "profile.h" -#if defined(_unix_) && !defined(_bionic_) && !defined(_cygwin_) +#if defined(_unix_) && !defined(_bionic_) && !defined(_cygwin_) #include <signal.h> #include <sys/time.h> @@ -12,22 +12,22 @@ #else #include <ucontext.h> #endif -#include <dlfcn.h> - +#include <dlfcn.h> + #include <util/system/platform.h> #include <util/generic/hash.h> #include <util/generic/map.h> -#include <util/generic/noncopyable.h> -#include <util/generic/algorithm.h> +#include <util/generic/noncopyable.h> +#include <util/generic/algorithm.h> #include <util/generic/vector.h> #include <util/stream/file.h> #include <util/string/util.h> #include <util/system/datetime.h> // This class sets SIGPROF handler and captures instruction pointer in it. -class TExecutionSampler : TNonCopyable { +class TExecutionSampler : TNonCopyable { public: - typedef TVector<std::pair<void*, size_t>> TSampleVector; + typedef TVector<std::pair<void*, size_t>> TSampleVector; struct TStats { ui64 SavedSamples; @@ -37,7 +37,7 @@ public: // NOTE: There is no synchronization here as the instance is supposed to be // created on the main thread. - static TExecutionSampler* Instance() { + static TExecutionSampler* Instance() { if (SInstance == nullptr) { SInstance = new TExecutionSampler(); } @@ -45,7 +45,7 @@ public: return SInstance; } - void Start() { + void Start() { // Set signal handler struct sigaction sa; sa.sa_sigaction = ProfilerSignalHandler; @@ -63,7 +63,7 @@ public: Started = true; } - void Stop(TSampleVector& sampleVector, TStats& stats) { + void Stop(TSampleVector& sampleVector, TStats& stats) { // Reset signal handler and timer if (Started) { setitimer(ITIMER_PROF, &OldTimerValue, nullptr); @@ -94,19 +94,19 @@ public: sampleVector.swap(hits); } - void ResetStats() { + void ResetStats() { WaitForWriteFlag(); Clear(); AtomicUnlock(&WriteFlag); } private: - static const size_t SZ = 2 * 1024 * 1024; // size of the hash table - // inserts work faster if it's a power of 2 - static const int SAMPLE_INTERVAL = 1000; // in microseconds + static const size_t SZ = 2 * 1024 * 1024; // size of the hash table + // inserts work faster if it's a power of 2 + static const int SAMPLE_INTERVAL = 1000; // in microseconds struct TCompareFirst { - bool operator()(const std::pair<void*, size_t>& a, const std::pair<void*, size_t>& b) const { + bool operator()(const std::pair<void*, size_t>& a, const std::pair<void*, size_t>& b) const { return a.first < b.first; } }; @@ -129,7 +129,7 @@ private: // were in process of update. // One such thing is memory allocation. That's why a fixed size vector is // preallocated at start. - static void ProfilerSignalHandler(int signal, siginfo_t* info, void* context) { + static void ProfilerSignalHandler(int signal, siginfo_t* info, void* context) { (void)info; if (signal != SIGPROF) { return; @@ -141,17 +141,17 @@ private: SInstance->CaptureIP(GetIp(&ucontext->uc_mcontext)); } - void WaitForWriteFlag() { + void WaitForWriteFlag() { // Wait for write flag to be reset ui32 delay = 100; while (!AtomicTryLock(&WriteFlag)) { usleep(delay); delay += delay; - delay = Min(delay, (ui32)5000); + delay = Min(delay, (ui32)5000); } } - void CaptureIP(void* rip) { + void CaptureIP(void* rip) { // Check if the handler on another thread is in the process of adding a sample // If this is the case, we just drop the current sample as this should happen // rarely. @@ -159,51 +159,51 @@ private: AddSample(rip); AtomicUnlock(&WriteFlag); } else { - AtomicIncrement(DroppedSamples); + AtomicIncrement(DroppedSamples); } } // Hash function applied to the addresses - static inline ui32 Hash(void* key) { + static inline ui32 Hash(void* key) { return ((size_t)key + (size_t)key / SZ) % SZ; } // Get instruction pointer from the context - static inline void* GetIp(const mcontext_t* mctx) { + static inline void* GetIp(const mcontext_t* mctx) { #if defined _freebsd_ -#if defined _64_ +#if defined _64_ return (void*)mctx->mc_rip; -#else +#else return (void*)mctx->mc_eip; -#endif +#endif #elif defined _linux_ -#if defined _64_ -#if defined(_arm_) - return (void*)mctx->pc; -#else +#if defined _64_ +#if defined(_arm_) + return (void*)mctx->pc; +#else return (void*)mctx->gregs[REG_RIP]; -#endif -#else +#endif +#else return (void*)mctx->gregs[REG_EIP]; -#endif +#endif #elif defined _darwin_ -#if defined _64_ +#if defined _64_ #if defined(_arm_) return (void*)(*mctx)->__ss.__pc; #else return (void*)(*mctx)->__ss.__rip; #endif -#else -#if defined(__IOS__) - return (void*)(*mctx)->__ss.__pc; -#else +#else +#if defined(__IOS__) + return (void*)(*mctx)->__ss.__pc; +#else return (void*)(*mctx)->__ss.__eip; +#endif #endif -#endif -#endif +#endif } - inline bool AddSample(void* key) { + inline bool AddSample(void* key) { ui32 slot = Hash(key); ui32 prevSlot = (slot - 1) % SZ; @@ -224,23 +224,23 @@ private: ++Samples; } else { // don't insert new sample if the search is becoming too slow - AtomicIncrement(DroppedSamples); + AtomicIncrement(DroppedSamples); return false; } return true; } - inline bool IsSlotEmpty(ui32 slot) const { + inline bool IsSlotEmpty(ui32 slot) const { return Ips[slot].first == nullptr; } - inline bool InsertsAllowed() const { + inline bool InsertsAllowed() const { return UniqueSamples < SZ / 2; } void - Clear() { + Clear() { Y_ASSERT(WriteFlag == 1); for (size_t i = 0; i < SZ; ++i) { @@ -252,21 +252,21 @@ private: SearchSkipCount = 0; } - bool Started; - struct sigaction OldSignalHandler; - itimerval OldTimerValue; + bool Started; + struct sigaction OldSignalHandler; + itimerval OldTimerValue; - TVector<std::pair<void*, size_t>> - Ips; // The hash table storing addresses and their hitcounts + TVector<std::pair<void*, size_t>> + Ips; // The hash table storing addresses and their hitcounts // TODO: on a big multiproc cache line false sharing by the flag and count might become an issue - TAtomic WriteFlag; // Is used to syncronize access to the hash table - TAtomic DroppedSamples; // "dropped sample" count will show how many times - // a sample was dropped either because of write conflict - // or because of the hash table had become too filled up - ui64 Samples; // Saved samples count - ui64 UniqueSamples; // Number of unique addresses - ui64 SearchSkipCount; // Total number of linear hash table probes due to collisions + TAtomic WriteFlag; // Is used to syncronize access to the hash table + TAtomic DroppedSamples; // "dropped sample" count will show how many times + // a sample was dropped either because of write conflict + // or because of the hash table had become too filled up + ui64 Samples; // Saved samples count + ui64 UniqueSamples; // Number of unique addresses + ui64 SearchSkipCount; // Total number of linear hash table probes due to collisions static TExecutionSampler* SInstance; }; @@ -288,14 +288,14 @@ public: private: TExecutionSampler::TSampleVector Samples; - TExecutionSampler::TStats Stats; + TExecutionSampler::TStats Stats; bool PutTimestamps; }; -void TSampleAnalyser::Analyze(FILE* out) const { +void TSampleAnalyser::Analyze(FILE* out) const { fprintf(out, "samples: %" PRIu64 " unique: %" PRIu64 " dropped: %" PRIu64 " searchskips: %" PRIu64 "\n", - (ui64)Stats.SavedSamples, (ui64)Samples.size(), - (ui64)Stats.DroppedSamples, (ui64)Stats.SearchSkipCount); + (ui64)Stats.SavedSamples, (ui64)Samples.size(), + (ui64)Stats.DroppedSamples, (ui64)Stats.SearchSkipCount); fprintf(out, "\nSamples:\n"); size_t funcCnt = 0; @@ -313,11 +313,11 @@ void TSampleAnalyser::Analyze(FILE* out) const { if (dladdr(Samples[i].first, &addrInfo)) { if (addrInfo.dli_fbase != prevModBase || addrInfo.dli_saddr != prevFunc) { fprintf(out, "Func\t%" PRISZT "\t%p\t%p\t%s\t%s\n", - funcCnt, - addrInfo.dli_fbase, - addrInfo.dli_saddr, - addrInfo.dli_fname, - addrInfo.dli_sname); + funcCnt, + addrInfo.dli_fbase, + addrInfo.dli_saddr, + addrInfo.dli_fname, + addrInfo.dli_sname); prevModBase = addrInfo.dli_fbase; prevFunc = addrInfo.dli_saddr; ++funcCnt; @@ -348,30 +348,30 @@ void DumpRUsage(FILE* out) { return; fprintf(out, - "user time: %lf\n" - "system time: %lf\n" - "max RSS: %ld\n" - "shared text: %ld\n" - "unshared data: %ld\n" - "unshared stack: %ld\n" - "page reclaims: %ld\n" - "page faults: %ld\n" - "swaps: %ld\n" - "block input ops: %ld\n" - "block output ops: %ld\n" - "msg sent: %ld\n" - "msg received: %ld\n" - "signals: %ld\n" - "voluntary ctx switches: %ld\n" - "involuntary ctx switches: %ld\n\n", - ru.ru_utime.tv_sec + (ru.ru_utime.tv_usec * 0.000001), - ru.ru_stime.tv_sec + (ru.ru_stime.tv_usec * 0.000001), - ru.ru_maxrss, ru.ru_ixrss, ru.ru_idrss, ru.ru_isrss, - ru.ru_minflt, ru.ru_majflt, ru.ru_nswap, - ru.ru_inblock, ru.ru_oublock, - ru.ru_msgsnd, ru.ru_msgrcv, - ru.ru_nsignals, - ru.ru_nvcsw, ru.ru_nivcsw); + "user time: %lf\n" + "system time: %lf\n" + "max RSS: %ld\n" + "shared text: %ld\n" + "unshared data: %ld\n" + "unshared stack: %ld\n" + "page reclaims: %ld\n" + "page faults: %ld\n" + "swaps: %ld\n" + "block input ops: %ld\n" + "block output ops: %ld\n" + "msg sent: %ld\n" + "msg received: %ld\n" + "signals: %ld\n" + "voluntary ctx switches: %ld\n" + "involuntary ctx switches: %ld\n\n", + ru.ru_utime.tv_sec + (ru.ru_utime.tv_usec * 0.000001), + ru.ru_stime.tv_sec + (ru.ru_stime.tv_usec * 0.000001), + ru.ru_maxrss, ru.ru_ixrss, ru.ru_idrss, ru.ru_isrss, + ru.ru_minflt, ru.ru_majflt, ru.ru_nswap, + ru.ru_inblock, ru.ru_oublock, + ru.ru_msgsnd, ru.ru_msgrcv, + ru.ru_nsignals, + ru.ru_nvcsw, ru.ru_nivcsw); } // Pauses capturing execution samples and dumps them to the file @@ -398,7 +398,7 @@ void EndProfiling() { ++cnt; } -#else +#else // NOTE: not supported on Windows @@ -408,10 +408,10 @@ void BeginProfiling() { void ResetProfile() { } -void EndProfiling(FILE*) { +void EndProfiling(FILE*) { } void EndProfiling() { } -#endif +#endif diff --git a/library/cpp/execprofile/ya.make b/library/cpp/execprofile/ya.make index 9d202ac4eb..39e4ad9c6a 100644 --- a/library/cpp/execprofile/ya.make +++ b/library/cpp/execprofile/ya.make @@ -1,6 +1,6 @@ OWNER(g:cpp-contrib) -LIBRARY() +LIBRARY() SRCS( profile.cpp diff --git a/library/cpp/getopt/last_getopt.h b/library/cpp/getopt/last_getopt.h index d14f05cc5b..55cb627970 100644 --- a/library/cpp/getopt/last_getopt.h +++ b/library/cpp/getopt/last_getopt.h @@ -1,3 +1,3 @@ #pragma once - + #include <library/cpp/getopt/small/last_getopt.h> diff --git a/library/cpp/getopt/last_getopt_demo/demo.cpp b/library/cpp/getopt/last_getopt_demo/demo.cpp index 79426a9cc9..e6a927c0a3 100644 --- a/library/cpp/getopt/last_getopt_demo/demo.cpp +++ b/library/cpp/getopt/last_getopt_demo/demo.cpp @@ -47,7 +47,7 @@ protected: void RegisterOptions(NLastGetopt::TOpts& opts) override { // Brief description for the whole program, will appear in the beginning of a help message. opts.SetTitle("last_getopt_demo -- like wget, but doesn't actually do anything"); - + // Built-in options. opts.AddHelpOption('h'); opts.AddCompletionOption("last_getopt_demo"); diff --git a/library/cpp/getopt/last_getopt_support.h b/library/cpp/getopt/last_getopt_support.h index b71c7045b1..29a656d02a 100644 --- a/library/cpp/getopt/last_getopt_support.h +++ b/library/cpp/getopt/last_getopt_support.h @@ -1,3 +1,3 @@ #pragma once - + #include <library/cpp/getopt/small/last_getopt_support.h> diff --git a/library/cpp/getopt/modchooser.h b/library/cpp/getopt/modchooser.h index 9bf73daf13..26b9a38614 100644 --- a/library/cpp/getopt/modchooser.h +++ b/library/cpp/getopt/modchooser.h @@ -1,3 +1,3 @@ #pragma once - + #include <library/cpp/getopt/small/modchooser.h> diff --git a/library/cpp/getopt/opt.h b/library/cpp/getopt/opt.h index d2a85075bd..2afd477e9c 100644 --- a/library/cpp/getopt/opt.h +++ b/library/cpp/getopt/opt.h @@ -1,3 +1,3 @@ #pragma once - + #include <library/cpp/getopt/small/opt.h> diff --git a/library/cpp/getopt/opt2.h b/library/cpp/getopt/opt2.h index 2d35bc7b29..fbe0d31b7c 100644 --- a/library/cpp/getopt/opt2.h +++ b/library/cpp/getopt/opt2.h @@ -1,3 +1,3 @@ #pragma once - + #include <library/cpp/getopt/small/opt2.h> diff --git a/library/cpp/getopt/posix_getopt.h b/library/cpp/getopt/posix_getopt.h index 8cb7ece624..75d5f235b9 100644 --- a/library/cpp/getopt/posix_getopt.h +++ b/library/cpp/getopt/posix_getopt.h @@ -1,3 +1,3 @@ #pragma once - + #include <library/cpp/getopt/small/posix_getopt.h> diff --git a/library/cpp/getopt/print.cpp b/library/cpp/getopt/print.cpp index 8cf1c62e4d..b49f7aa99f 100644 --- a/library/cpp/getopt/print.cpp +++ b/library/cpp/getopt/print.cpp @@ -1,25 +1,25 @@ #include "last_getopt.h" -#include "last_getopt_support.h" -#include "modchooser.h" -#include "opt.h" -#include "opt2.h" -#include "posix_getopt.h" -#include "ygetopt.h" +#include "last_getopt_support.h" +#include "modchooser.h" +#include "opt.h" +#include "opt2.h" +#include "posix_getopt.h" +#include "ygetopt.h" #include <library/cpp/svnversion/svnversion.h> #include <library/cpp/build_info/build_info.h> namespace NLastGetoptPrivate { TString InitVersionString() { - TString ts = GetProgramSvnVersion(); - ts += "\n"; - ts += GetBuildInfo(); - TString sandboxTaskId = GetSandboxTaskId(); - if (sandboxTaskId != TString("0")) { - ts += "\nSandbox task id: "; - ts += sandboxTaskId; - } - return ts; + TString ts = GetProgramSvnVersion(); + ts += "\n"; + ts += GetBuildInfo(); + TString sandboxTaskId = GetSandboxTaskId(); + if (sandboxTaskId != TString("0")) { + ts += "\nSandbox task id: "; + ts += sandboxTaskId; + } + return ts; } TString InitShortVersionString() { diff --git a/library/cpp/getopt/small/last_getopt.cpp b/library/cpp/getopt/small/last_getopt.cpp index 30669b2c5a..42b1affef1 100644 --- a/library/cpp/getopt/small/last_getopt.cpp +++ b/library/cpp/getopt/small/last_getopt.cpp @@ -1,9 +1,9 @@ -#include "last_getopt.h" - +#include "last_getopt.h" + namespace NLastGetopt { - void PrintUsageAndExit(const TOptsParser* parser) { - parser->PrintUsage(); - exit(0); - } + void PrintUsageAndExit(const TOptsParser* parser) { + parser->PrintUsage(); + exit(0); + } } diff --git a/library/cpp/getopt/small/last_getopt.h b/library/cpp/getopt/small/last_getopt.h index 07687bc914..c472d6ea2c 100644 --- a/library/cpp/getopt/small/last_getopt.h +++ b/library/cpp/getopt/small/last_getopt.h @@ -18,7 +18,7 @@ namespace NLastGetopt { /// Handler to split option value by delimiter into a target container and allow ranges. template <class Container> - struct TOptRangeSplitHandler: public IOptHandler { + struct TOptRangeSplitHandler: public IOptHandler { public: using TContainer = Container; using TValue = typename TContainer::value_type; @@ -57,7 +57,7 @@ namespace NLastGetopt { }; template <class Container> - struct TOptSplitHandler: public IOptHandler { + struct TOptSplitHandler: public IOptHandler { public: using TContainer = Container; using TValue = typename TContainer::value_type; @@ -83,7 +83,7 @@ namespace NLastGetopt { }; template <class TpFunc> - struct TOptKVHandler: public IOptHandler { + struct TOptKVHandler: public IOptHandler { public: using TKey = typename TFunctionArgs<TpFunc>::template TGet<0>; using TValue = typename TFunctionArgs<TpFunc>::template TGet<1>; @@ -101,7 +101,7 @@ namespace NLastGetopt { TStringBuf key, value; if (!curval.TrySplit(KVDelim, key, value)) { throw TUsageException() << "failed to parse opt " << NPrivate::OptToString(curOpt) - << " value " << TString(curval).Quote() << ": expected key" << KVDelim << "value format"; + << " value " << TString(curval).Quote() << ": expected key" << KVDelim << "value format"; } Func(NPrivate::OptFromString<TKey>(key, curOpt), NPrivate::OptFromString<TValue>(value, curOpt)); } @@ -112,21 +112,21 @@ namespace NLastGetopt { char KVDelim; }; - namespace NPrivate { - template <typename TpFunc, typename TpArg> - void THandlerFunctor1<TpFunc, TpArg>::HandleOpt(const TOptsParser* parser) { - const TStringBuf curval = parser->CurValOrDef(!HasDef_); - const TpArg& arg = curval.IsInited() ? OptFromString<TpArg>(curval, parser->CurOpt()) : Def_; - try { - Func_(arg); + namespace NPrivate { + template <typename TpFunc, typename TpArg> + void THandlerFunctor1<TpFunc, TpArg>::HandleOpt(const TOptsParser* parser) { + const TStringBuf curval = parser->CurValOrDef(!HasDef_); + const TpArg& arg = curval.IsInited() ? OptFromString<TpArg>(curval, parser->CurOpt()) : Def_; + try { + Func_(arg); } catch (const TUsageException&) { throw; - } catch (...) { + } catch (...) { throw TUsageException() << "failed to handle opt " << OptToString(parser->CurOpt()) - << " value " << TString(curval).Quote() << ": " << CurrentExceptionMessage(); - } - } + << " value " << TString(curval).Quote() << ": " << CurrentExceptionMessage(); + } + } } - + } diff --git a/library/cpp/getopt/small/last_getopt_easy_setup.cpp b/library/cpp/getopt/small/last_getopt_easy_setup.cpp index c87dedf95e..f2a93056e4 100644 --- a/library/cpp/getopt/small/last_getopt_easy_setup.cpp +++ b/library/cpp/getopt/small/last_getopt_easy_setup.cpp @@ -1,47 +1,47 @@ #include "last_getopt_easy_setup.h" namespace NLastGetopt { - TEasySetup::TEasySetup(const TStringBuf& optstring) - : TOpts(optstring) - { - AddHelpOption(); - } + TEasySetup::TEasySetup(const TStringBuf& optstring) + : TOpts(optstring) + { + AddHelpOption(); + } - TOpt& TEasySetup::AdjustParam(const char* longName, const char* help, const char* argName, bool required) { - Y_ASSERT(longName); - TOpt& o = AddLongOption(longName); - if (help) { - o.Help(help); - } - if (argName) { - o.RequiredArgument(argName); - } else { - o.HasArg(NO_ARGUMENT); - } - if (required) { - o.Required(); - } - return o; - } + TOpt& TEasySetup::AdjustParam(const char* longName, const char* help, const char* argName, bool required) { + Y_ASSERT(longName); + TOpt& o = AddLongOption(longName); + if (help) { + o.Help(help); + } + if (argName) { + o.RequiredArgument(argName); + } else { + o.HasArg(NO_ARGUMENT); + } + if (required) { + o.Required(); + } + return o; + } - TEasySetup& TEasySetup::operator()(char shortName, const char* longName, const char* help, bool required) { - AdjustParam(longName, help, nullptr, required).AddShortName(shortName); - return *this; + TEasySetup& TEasySetup::operator()(char shortName, const char* longName, const char* help, bool required) { + AdjustParam(longName, help, nullptr, required).AddShortName(shortName); + return *this; } - - TEasySetup& TEasySetup::operator()(char shortName, const char* longName, const char* argName, const char* help, bool required) { - AdjustParam(longName, help, argName, required).AddShortName(shortName); - return *this; + + TEasySetup& TEasySetup::operator()(char shortName, const char* longName, const char* argName, const char* help, bool required) { + AdjustParam(longName, help, argName, required).AddShortName(shortName); + return *this; } - - TEasySetup& TEasySetup::operator()(const char* longName, const char* help, bool required) { - AdjustParam(longName, help, nullptr, required); - return *this; + + TEasySetup& TEasySetup::operator()(const char* longName, const char* help, bool required) { + AdjustParam(longName, help, nullptr, required); + return *this; } - TEasySetup& TEasySetup::operator()(const char* longName, const char* argName, const char* help, bool required) { - AdjustParam(longName, help, argName, required); - return *this; - } + TEasySetup& TEasySetup::operator()(const char* longName, const char* argName, const char* help, bool required) { + AdjustParam(longName, help, argName, required); + return *this; + } } diff --git a/library/cpp/getopt/small/last_getopt_easy_setup.h b/library/cpp/getopt/small/last_getopt_easy_setup.h index 60dddda225..f5fabad227 100644 --- a/library/cpp/getopt/small/last_getopt_easy_setup.h +++ b/library/cpp/getopt/small/last_getopt_easy_setup.h @@ -3,7 +3,7 @@ #include "last_getopt_opts.h" namespace NLastGetopt { - /** + /** * Wrapper for TOpts class to make the life a bit easier. * Usual usage: * TEasySetup opts; @@ -16,36 +16,36 @@ namespace NLastGetopt { * * NLastGetopt::TOptsParseResult r(&opts, argc, argv); */ - class TEasySetup: public TOpts { - public: - TEasySetup(const TStringBuf& optstring = TStringBuf()); - TEasySetup& operator()(char shortName, const char* longName, const char* help, bool required = false); - TEasySetup& operator()(char shortName, const char* longName, const char* argName, const char* help, bool required = false); - - template <class TpFunc> - TEasySetup& operator()(char shortName, const char* longName, TpFunc handler, const char* help, bool required = false) { - AdjustParam(longName, help, nullptr, handler, required).AddShortName(shortName); - return *this; - } - - TEasySetup& operator()(const char* longName, const char* help, bool required = false); - TEasySetup& operator()(const char* longName, const char* argName, const char* help, bool required = false); - - template <class TpFunc> - TEasySetup& operator()(const char* longName, TpFunc handler, const char* help, bool required = false) { - AdjustParam(longName, help, nullptr, handler, required); - return *this; - } - - private: - TOpt& AdjustParam(const char* longName, const char* help, const char* argName, bool required); - - template <class TpFunc> - TOpt& AdjustParam(const char* longName, const char* help, const char* argName, TpFunc handler, bool required) { - TOpt& o = AdjustParam(longName, help, argName, required); - o.Handler0(handler); - return o; - } - }; + class TEasySetup: public TOpts { + public: + TEasySetup(const TStringBuf& optstring = TStringBuf()); + TEasySetup& operator()(char shortName, const char* longName, const char* help, bool required = false); + TEasySetup& operator()(char shortName, const char* longName, const char* argName, const char* help, bool required = false); + + template <class TpFunc> + TEasySetup& operator()(char shortName, const char* longName, TpFunc handler, const char* help, bool required = false) { + AdjustParam(longName, help, nullptr, handler, required).AddShortName(shortName); + return *this; + } + + TEasySetup& operator()(const char* longName, const char* help, bool required = false); + TEasySetup& operator()(const char* longName, const char* argName, const char* help, bool required = false); + + template <class TpFunc> + TEasySetup& operator()(const char* longName, TpFunc handler, const char* help, bool required = false) { + AdjustParam(longName, help, nullptr, handler, required); + return *this; + } + + private: + TOpt& AdjustParam(const char* longName, const char* help, const char* argName, bool required); + + template <class TpFunc> + TOpt& AdjustParam(const char* longName, const char* help, const char* argName, TpFunc handler, bool required) { + TOpt& o = AdjustParam(longName, help, argName, required); + o.Handler0(handler); + return o; + } + }; } diff --git a/library/cpp/getopt/small/last_getopt_opt.cpp b/library/cpp/getopt/small/last_getopt_opt.cpp index 9a99437f4b..576b8f03d5 100644 --- a/library/cpp/getopt/small/last_getopt_opt.cpp +++ b/library/cpp/getopt/small/last_getopt_opt.cpp @@ -11,95 +11,95 @@ namespace NLastGetopt { static const TStringBuf ExcludedShortNameChars = "= -\t\n"; static const TStringBuf ExcludedLongNameChars = "= \t\n"; - bool TOpt::NameIs(const TString& name) const { - for (const auto& next : LongNames_) { - if (next == name) - return true; - } - return false; - } - - bool TOpt::CharIs(char c) const { - for (auto next : Chars_) { - if (next == c) - return true; - } - return false; - } - - char TOpt::GetChar() const { - ; - if (Chars_.empty()) - ythrow TConfException() << "no char for option " << this->ToShortString(); - return Chars_.at(0); + bool TOpt::NameIs(const TString& name) const { + for (const auto& next : LongNames_) { + if (next == name) + return true; + } + return false; + } + + bool TOpt::CharIs(char c) const { + for (auto next : Chars_) { + if (next == c) + return true; + } + return false; + } + + char TOpt::GetChar() const { + ; + if (Chars_.empty()) + ythrow TConfException() << "no char for option " << this->ToShortString(); + return Chars_.at(0); } - char TOpt::GetCharOr0() const { - if (Chars_.empty()) - return 0; - return GetChar(); + char TOpt::GetCharOr0() const { + if (Chars_.empty()) + return 0; + return GetChar(); } - TString TOpt::GetName() const { - ; - if (LongNames_.empty()) - ythrow TConfException() << "no name for option " << this->ToShortString(); - return LongNames_.at(0); - } + TString TOpt::GetName() const { + ; + if (LongNames_.empty()) + ythrow TConfException() << "no name for option " << this->ToShortString(); + return LongNames_.at(0); + } - bool TOpt::IsAllowedShortName(unsigned char c) { - return isprint(c) && TStringBuf::npos == ExcludedShortNameChars.find(c); - } + bool TOpt::IsAllowedShortName(unsigned char c) { + return isprint(c) && TStringBuf::npos == ExcludedShortNameChars.find(c); + } - TOpt& TOpt::AddShortName(unsigned char c) { - ; - if (!IsAllowedShortName(c)) + TOpt& TOpt::AddShortName(unsigned char c) { + ; + if (!IsAllowedShortName(c)) throw TUsageException() << "option char '" << c << "' is not allowed"; - Chars_.push_back(c); - return *this; - } - - bool TOpt::IsAllowedLongName(const TString& name, unsigned char* out) { - for (size_t i = 0; i != name.size(); ++i) { - const unsigned char c = name[i]; - if (!isprint(c) || TStringBuf::npos != ExcludedLongNameChars.find(c)) { - if (nullptr != out) - *out = c; - return false; - } + Chars_.push_back(c); + return *this; + } + + bool TOpt::IsAllowedLongName(const TString& name, unsigned char* out) { + for (size_t i = 0; i != name.size(); ++i) { + const unsigned char c = name[i]; + if (!isprint(c) || TStringBuf::npos != ExcludedLongNameChars.find(c)) { + if (nullptr != out) + *out = c; + return false; + } } - return true; + return true; } - TOpt& TOpt::AddLongName(const TString& name) { - ; - unsigned char c = 0; - if (!IsAllowedLongName(name, &c)) + TOpt& TOpt::AddLongName(const TString& name) { + ; + unsigned char c = 0; + if (!IsAllowedLongName(name, &c)) throw TUsageException() << "option char '" << c - << "' in long '" << name << "' is not allowed"; - LongNames_.push_back(name); - return *this; - } - - namespace NPrivate { - TString OptToString(char c); - - TString OptToString(const TString& longOption); - } - - TString TOpt::ToShortString() const { - ; - if (!LongNames_.empty()) - return NPrivate::OptToString(LongNames_.front()); - if (!Chars_.empty()) - return NPrivate::OptToString(Chars_.front()); - return "?"; - } - - void TOpt::FireHandlers(const TOptsParser* parser) const { - for (const auto& handler : Handlers_) { - handler->HandleOpt(parser); - } + << "' in long '" << name << "' is not allowed"; + LongNames_.push_back(name); + return *this; + } + + namespace NPrivate { + TString OptToString(char c); + + TString OptToString(const TString& longOption); + } + + TString TOpt::ToShortString() const { + ; + if (!LongNames_.empty()) + return NPrivate::OptToString(LongNames_.front()); + if (!Chars_.empty()) + return NPrivate::OptToString(Chars_.front()); + return "?"; + } + + void TOpt::FireHandlers(const TOptsParser* parser) const { + for (const auto& handler : Handlers_) { + handler->HandleOpt(parser); + } } TOpt& TOpt::IfPresentDisableCompletionFor(const TOpt& opt) { diff --git a/library/cpp/getopt/small/last_getopt_opt.h b/library/cpp/getopt/small/last_getopt_opt.h index a8dd5adca9..fbd393a310 100644 --- a/library/cpp/getopt/small/last_getopt_opt.h +++ b/library/cpp/getopt/small/last_getopt_opt.h @@ -13,14 +13,14 @@ #include <stdarg.h> namespace NLastGetopt { - enum EHasArg { - NO_ARGUMENT, - REQUIRED_ARGUMENT, - OPTIONAL_ARGUMENT, - DEFAULT_HAS_ARG = REQUIRED_ARGUMENT - }; - - /** + enum EHasArg { + NO_ARGUMENT, + REQUIRED_ARGUMENT, + OPTIONAL_ARGUMENT, + DEFAULT_HAS_ARG = REQUIRED_ARGUMENT + }; + + /** * NLastGetopt::TOpt is a storage of data about exactly one program option. * The data is: parse politics and help information. * @@ -40,28 +40,28 @@ namespace NLastGetopt { * in case of "not given <optional value>, omited optional argument" the <default value> is used * user value: allows to store arbitary pointer for handlers */ - class TOpt { - public: - typedef TVector<char> TShortNames; - typedef TVector<TString> TLongNames; + class TOpt { + public: + typedef TVector<char> TShortNames; + typedef TVector<TString> TLongNames; - protected: - TShortNames Chars_; - TLongNames LongNames_; + protected: + TShortNames Chars_; + TLongNames LongNames_; - private: - typedef TMaybe<TString> TdOptVal; - typedef TVector<TSimpleSharedPtr<IOptHandler>> TOptHandlers; + private: + typedef TMaybe<TString> TdOptVal; + typedef TVector<TSimpleSharedPtr<IOptHandler>> TOptHandlers; - public: + public: bool Hidden_ = false; // is visible in help TString ArgTitle_; // the name of argument in help output TString Help_; // the help string TString CompletionHelp_; // the help string that's used in completion script, a shorter version of Help_ TString CompletionArgHelp_; // the description of argument in completion script - EHasArg HasArg_ = DEFAULT_HAS_ARG; // the argument parsing politics - bool Required_ = false; // option existence politics + EHasArg HasArg_ = DEFAULT_HAS_ARG; // the argument parsing politics + bool Required_ = false; // option existence politics bool AllowMultipleCompletion_ = false; // let the completer know that this option can occur more than once @@ -72,104 +72,104 @@ namespace NLastGetopt { TVector<size_t> DisableCompletionForFreeArg_; NComp::ICompleterPtr Completer_; - private: - //Handlers information - const void* UserValue_ = nullptr; - TdOptVal OptionalValue_; - TdOptVal DefaultValue_; - TOptHandlers Handlers_; + private: + //Handlers information + const void* UserValue_ = nullptr; + TdOptVal OptionalValue_; + TdOptVal DefaultValue_; + TOptHandlers Handlers_; - public: - /** + public: + /** * Checks if given char can be a short name * @param c char to check */ - static bool IsAllowedShortName(unsigned char c); + static bool IsAllowedShortName(unsigned char c); - /** + /** * Checks if given string can be a long name * @param name string to check * @param c if given, the first bad charecter will be saved in c */ - static bool IsAllowedLongName(const TString& name, unsigned char* c = nullptr); + static bool IsAllowedLongName(const TString& name, unsigned char* c = nullptr); - /** + /** * @return one of the expected representations of the option. * If the option has short names, will return "-<char>" * Otherwise will return "--<long name>" */ - TString ToShortString() const; + TString ToShortString() const; - /** + /** * check if given string is one of the long names * * @param name string to check */ - bool NameIs(const TString& name) const; + bool NameIs(const TString& name) const; - /** + /** * check if given char is one of the short names * * @param c char to check */ - bool CharIs(char c) const; + bool CharIs(char c) const; - /** + /** * If string has long names - will return one of them * Otherwise will throw */ - TString GetName() const; + TString GetName() const; - /** + /** * adds short alias for the option * * @param c new short name * * @return self */ - TOpt& AddShortName(unsigned char c); + TOpt& AddShortName(unsigned char c); - /** + /** * return all short names of the option */ - const TShortNames& GetShortNames() const { - return Chars_; - } + const TShortNames& GetShortNames() const { + return Chars_; + } - /** + /** * adds long alias for the option * * @param name new long name * * @return self */ - TOpt& AddLongName(const TString& name); + TOpt& AddLongName(const TString& name); - /** + /** * return all long names of the option */ - const TLongNames& GetLongNames() const { - return LongNames_; - } + const TLongNames& GetLongNames() const { + return LongNames_; + } - /** + /** * @return one of short names of the opt. If there is no short names exception is raised. */ - char GetChar() const; + char GetChar() const; - /** + /** * @return one of short names of the opt. If there is no short names '\0' returned. */ - char GetCharOr0() const; + char GetCharOr0() const; - /** + /** * @returns argument parsing politics */ - const EHasArg& GetHasArg() const { - return HasArg_; - } + const EHasArg& GetHasArg() const { + return HasArg_; + } - /** + /** * sets argument parsing politics * * Note: its better use one of RequiredArgument/NoArgument/OptionalArgument methods @@ -177,51 +177,51 @@ namespace NLastGetopt { * @param hasArg new argument parsing mode * @return self */ - TOpt& HasArg(EHasArg hasArg) { - HasArg_ = hasArg; - return *this; - } + TOpt& HasArg(EHasArg hasArg) { + HasArg_ = hasArg; + return *this; + } - /** + /** * @returns argument title */ - TString GetArgTitle() const { - return ArgTitle_; - } + TString GetArgTitle() const { + return ArgTitle_; + } - /** + /** * sets argument parsing politics into REQUIRED_ARGUMENT * * @param title the new name of argument in help output * @return self */ - TOpt& RequiredArgument(const TString& title = "") { - ArgTitle_ = title; - return HasArg(REQUIRED_ARGUMENT); - } + TOpt& RequiredArgument(const TString& title = "") { + ArgTitle_ = title; + return HasArg(REQUIRED_ARGUMENT); + } - /** + /** * sets argument parsing politics into NO_ARGUMENT * * @return self */ - TOpt& NoArgument() { - return HasArg(NO_ARGUMENT); - } + TOpt& NoArgument() { + return HasArg(NO_ARGUMENT); + } - /** + /** * sets argument parsing politics into OPTIONAL_ARGUMENT * for details see NLastGetopt::TOpt * * @param title the new name of argument in help output * @return self */ - TOpt& OptionalArgument(const TString& title = "") { - ArgTitle_ = title; - return HasArg(OPTIONAL_ARGUMENT); - } + TOpt& OptionalArgument(const TString& title = "") { + ArgTitle_ = title; + return HasArg(OPTIONAL_ARGUMENT); + } - /** + /** * sets argument parsing politics into OPTIONAL_ARGUMENT * sets the <optional value> into given * @@ -231,110 +231,110 @@ namespace NLastGetopt { * @param title the new name of argument in help output * @return self */ - TOpt& OptionalValue(const TString& val, const TString& title = "") { - OptionalValue_ = val; - return OptionalArgument(title); - } + TOpt& OptionalValue(const TString& val, const TString& title = "") { + OptionalValue_ = val; + return OptionalArgument(title); + } - /** + /** * checks if "argument parsing politics" is OPTIONAL_ARGUMENT and the <optional value> is set. */ - bool HasOptionalValue() const { - return OPTIONAL_ARGUMENT == HasArg_ && OptionalValue_; - } + bool HasOptionalValue() const { + return OPTIONAL_ARGUMENT == HasArg_ && OptionalValue_; + } - /** + /** * @return optional value * throws exception if optional value wasn't set */ const TString& GetOptionalValue() const { - return *OptionalValue_; - } + return *OptionalValue_; + } - /** + /** * sets <default value> * @return self */ - template <typename T> - TOpt& DefaultValue(const T& val) { - DefaultValue_ = ToString(val); - return *this; - } + template <typename T> + TOpt& DefaultValue(const T& val) { + DefaultValue_ = ToString(val); + return *this; + } - /** + /** * checks if default value is set. */ - bool HasDefaultValue() const { - return DefaultValue_.Defined(); - } + bool HasDefaultValue() const { + return DefaultValue_.Defined(); + } - /** + /** * @return default value * throws exception if <default value> wasn't set */ const TString& GetDefaultValue() const { - return *DefaultValue_; - } + return *DefaultValue_; + } - /** + /** * sets the option to be required * @return self */ - TOpt& Required() { - Required_ = true; - return *this; - } + TOpt& Required() { + Required_ = true; + return *this; + } - /** + /** * sets the option to be optional * @return self */ - TOpt& Optional() { - Required_ = false; - return *this; - } + TOpt& Optional() { + Required_ = false; + return *this; + } - /** + /** * @return true if the option is required */ - bool IsRequired() const { - return Required_; - } + bool IsRequired() const { + return Required_; + } - /** + /** * sets the option to be hidden (invisible in help) * @return self */ - TOpt& Hidden() { - Hidden_ = true; - return *this; - } + TOpt& Hidden() { + Hidden_ = true; + return *this; + } - /** + /** * @return true if the option is hidden */ - bool IsHidden() const { - return Hidden_; - } + bool IsHidden() const { + return Hidden_; + } - /** + /** * sets the <user value> * @return self * for details see NLastGetopt::TOpt */ - TOpt& UserValue(const void* userval) { - UserValue_ = userval; - return *this; - } + TOpt& UserValue(const void* userval) { + UserValue_ = userval; + return *this; + } - /** + /** * @return user value */ - const void* UserValue() const { - return UserValue_; - } + const void* UserValue() const { + return UserValue_; + } - /** + /** * Set help string that appears with `--help`. Unless `CompletionHelp` is given, this message will also be used * in completion script. In this case, don't make it too long, don't start it with a capital letter and don't * end it with a full stop. @@ -367,19 +367,19 @@ namespace NLastGetopt { * See more on completion descriptions codestyle: * https://github.com/zsh-users/zsh/blob/master/Etc/completion-style-guide#L43 */ - TOpt& Help(const TString& help) { - Help_ = help; - return *this; - } + TOpt& Help(const TString& help) { + Help_ = help; + return *this; + } - /** + /** * Get help string. */ const TString& GetHelp() const { - return Help_; - } + return Help_; + } - /** + /** * Set help string that appears when argument completer lists available options. * * See `Help` function for info on how this is different from setting `Help` and `CompletionArgHelp`. @@ -545,64 +545,64 @@ namespace NLastGetopt { /** * Run handlers for this option. */ - void FireHandlers(const TOptsParser* parser) const; - - private: - TOpt& HandlerImpl(IOptHandler* handler) { - Handlers_.push_back(handler); - return *this; - } - - public: - template <typename TpFunc> - TOpt& Handler0(TpFunc func) { // functor taking no parameters - return HandlerImpl(new NPrivate::THandlerFunctor0<TpFunc>(func)); - } - - template <typename TpFunc> - TOpt& Handler1(TpFunc func) { // functor taking one parameter - return HandlerImpl(new NPrivate::THandlerFunctor1<TpFunc>(func)); - } - template <typename TpArg, typename TpFunc> - TOpt& Handler1T(TpFunc func) { - return HandlerImpl(new NPrivate::THandlerFunctor1<TpFunc, TpArg>(func)); - } - template <typename TpArg, typename TpFunc> - TOpt& Handler1T(const TpArg& def, TpFunc func) { - return HandlerImpl(new NPrivate::THandlerFunctor1<TpFunc, TpArg>(func, def)); - } - template <typename TpArg, typename TpArg2, typename TpFunc> - TOpt& Handler1T2(const TpArg2& def, TpFunc func) { - return HandlerImpl(new NPrivate::THandlerFunctor1<TpFunc, TpArg>(func, def)); - } - - TOpt& Handler(void (*f)()) { - return Handler0(f); - } - TOpt& Handler(void (*f)(const TOptsParser*)) { - return Handler1(f); - } - - TOpt& Handler(TAutoPtr<IOptHandler> handler) { - return HandlerImpl(handler.Release()); - } - - template <typename T> // T extends IOptHandler - TOpt& Handler(TAutoPtr<T> handler) { - return HandlerImpl(handler.Release()); - } - - // Stores FromString<T>(arg) in *target - // T maybe anything with FromString<T>(const TStringBuf&) defined - template <typename TpVal, typename T> - TOpt& StoreResultT(T* target) { - return Handler1T<TpVal>(NPrivate::TStoreResultFunctor<T, TpVal>(target)); - } - - template <typename T> - TOpt& StoreResult(T* target) { - return StoreResultT<T>(target); - } + void FireHandlers(const TOptsParser* parser) const; + + private: + TOpt& HandlerImpl(IOptHandler* handler) { + Handlers_.push_back(handler); + return *this; + } + + public: + template <typename TpFunc> + TOpt& Handler0(TpFunc func) { // functor taking no parameters + return HandlerImpl(new NPrivate::THandlerFunctor0<TpFunc>(func)); + } + + template <typename TpFunc> + TOpt& Handler1(TpFunc func) { // functor taking one parameter + return HandlerImpl(new NPrivate::THandlerFunctor1<TpFunc>(func)); + } + template <typename TpArg, typename TpFunc> + TOpt& Handler1T(TpFunc func) { + return HandlerImpl(new NPrivate::THandlerFunctor1<TpFunc, TpArg>(func)); + } + template <typename TpArg, typename TpFunc> + TOpt& Handler1T(const TpArg& def, TpFunc func) { + return HandlerImpl(new NPrivate::THandlerFunctor1<TpFunc, TpArg>(func, def)); + } + template <typename TpArg, typename TpArg2, typename TpFunc> + TOpt& Handler1T2(const TpArg2& def, TpFunc func) { + return HandlerImpl(new NPrivate::THandlerFunctor1<TpFunc, TpArg>(func, def)); + } + + TOpt& Handler(void (*f)()) { + return Handler0(f); + } + TOpt& Handler(void (*f)(const TOptsParser*)) { + return Handler1(f); + } + + TOpt& Handler(TAutoPtr<IOptHandler> handler) { + return HandlerImpl(handler.Release()); + } + + template <typename T> // T extends IOptHandler + TOpt& Handler(TAutoPtr<T> handler) { + return HandlerImpl(handler.Release()); + } + + // Stores FromString<T>(arg) in *target + // T maybe anything with FromString<T>(const TStringBuf&) defined + template <typename TpVal, typename T> + TOpt& StoreResultT(T* target) { + return Handler1T<TpVal>(NPrivate::TStoreResultFunctor<T, TpVal>(target)); + } + + template <typename T> + TOpt& StoreResult(T* target) { + return StoreResultT<T>(target); + } // Uses TMaybe<T> to store FromString<T>(arg) template <typename T> @@ -610,15 +610,15 @@ namespace NLastGetopt { return StoreResultT<T>(target); } - template <typename TpVal, typename T, typename TpDef> - TOpt& StoreResultT(T* target, const TpDef& def) { - return Handler1T<TpVal>(def, NPrivate::TStoreResultFunctor<T, TpVal>(target)); - } + template <typename TpVal, typename T, typename TpDef> + TOpt& StoreResultT(T* target, const TpDef& def) { + return Handler1T<TpVal>(def, NPrivate::TStoreResultFunctor<T, TpVal>(target)); + } - template <typename T, typename TpDef> - TOpt& StoreResult(T* target, const TpDef& def) { - return StoreResultT<T>(target, def); - } + template <typename T, typename TpDef> + TOpt& StoreResult(T* target, const TpDef& def) { + return StoreResultT<T>(target, def); + } template <typename T> TOpt& StoreResultDef(T* target) { @@ -632,51 +632,51 @@ namespace NLastGetopt { return StoreResultT<T>(target, def); } - // Sugar for storing flags (option without arguments) to boolean vars - TOpt& SetFlag(bool* target) { - return DefaultValue("0").StoreResult(target, true); - } + // Sugar for storing flags (option without arguments) to boolean vars + TOpt& SetFlag(bool* target) { + return DefaultValue("0").StoreResult(target, true); + } // Similar to store_true in Python's argparse TOpt& StoreTrue(bool* target) { return NoArgument().SetFlag(target); } - template <typename TpVal, typename T, typename TpFunc> - TOpt& StoreMappedResultT(T* target, const TpFunc& func) { - return Handler1T<TpVal>(NPrivate::TStoreMappedResultFunctor<T, TpFunc, TpVal>(target, func)); - } - - template <typename T, typename TpFunc> - TOpt& StoreMappedResult(T* target, const TpFunc& func) { - return StoreMappedResultT<T>(target, func); - } - - // Stores given value in *target if the option is present. - // TValue must be a copyable type, constructible from TParam. - // T must be a copyable type, assignable from TValue. - template <typename TValue, typename T, typename TParam> - TOpt& StoreValueT(T* target, const TParam& value) { - return Handler1(NPrivate::TStoreValueFunctor<T, TValue>(target, value)); - } - - // save value as target type - template <typename T, typename TParam> - TOpt& StoreValue(T* target, const TParam& value) { - return StoreValueT<T>(target, value); - } - - // save value as its original type (2nd template parameter) - template <typename T, typename TValue> - TOpt& StoreValue2(T* target, const TValue& value) { - return StoreValueT<TValue>(target, value); - } - - // Appends FromString<T>(arg) to *target for each argument - template <typename T> - TOpt& AppendTo(TVector<T>* target) { + template <typename TpVal, typename T, typename TpFunc> + TOpt& StoreMappedResultT(T* target, const TpFunc& func) { + return Handler1T<TpVal>(NPrivate::TStoreMappedResultFunctor<T, TpFunc, TpVal>(target, func)); + } + + template <typename T, typename TpFunc> + TOpt& StoreMappedResult(T* target, const TpFunc& func) { + return StoreMappedResultT<T>(target, func); + } + + // Stores given value in *target if the option is present. + // TValue must be a copyable type, constructible from TParam. + // T must be a copyable type, assignable from TValue. + template <typename TValue, typename T, typename TParam> + TOpt& StoreValueT(T* target, const TParam& value) { + return Handler1(NPrivate::TStoreValueFunctor<T, TValue>(target, value)); + } + + // save value as target type + template <typename T, typename TParam> + TOpt& StoreValue(T* target, const TParam& value) { + return StoreValueT<T>(target, value); + } + + // save value as its original type (2nd template parameter) + template <typename T, typename TValue> + TOpt& StoreValue2(T* target, const TValue& value) { + return StoreValueT<TValue>(target, value); + } + + // Appends FromString<T>(arg) to *target for each argument + template <typename T> + TOpt& AppendTo(TVector<T>* target) { return Handler1T<T>([target](auto&& value) { target->push_back(std::move(value)); }); - } + } // Appends FromString<T>(arg) to *target for each argument template <typename T> @@ -690,23 +690,23 @@ namespace NLastGetopt { return Handler1T<TString>([target](TString arg) { target->emplace_back(std::move(arg)); } ); } - template <class Container> - TOpt& SplitHandler(Container* target, const char delim) { - return Handler(new NLastGetopt::TOptSplitHandler<Container>(target, delim)); - } + template <class Container> + TOpt& SplitHandler(Container* target, const char delim) { + return Handler(new NLastGetopt::TOptSplitHandler<Container>(target, delim)); + } - template <class Container> - TOpt& RangeSplitHandler(Container* target, const char elementsDelim, const char rangesDelim) { - return Handler(new NLastGetopt::TOptRangeSplitHandler<Container>(target, elementsDelim, rangesDelim)); - } + template <class Container> + TOpt& RangeSplitHandler(Container* target, const char elementsDelim, const char rangesDelim) { + return Handler(new NLastGetopt::TOptRangeSplitHandler<Container>(target, elementsDelim, rangesDelim)); + } - template <class TpFunc> - TOpt& KVHandler(TpFunc func, const char kvdelim = '=') { - return Handler(new NLastGetopt::TOptKVHandler<TpFunc>(func, kvdelim)); - } - }; + template <class TpFunc> + TOpt& KVHandler(TpFunc func, const char kvdelim = '=') { + return Handler(new NLastGetopt::TOptKVHandler<TpFunc>(func, kvdelim)); + } + }; - /** + /** * NLastGetopt::TFreeArgSpec is a storage of data about free argument. * The data is help information and (maybe) linked named argument. * @@ -714,14 +714,14 @@ namespace NLastGetopt { * help string * argument name (title) */ - struct TFreeArgSpec { - TFreeArgSpec() = default; + struct TFreeArgSpec { + TFreeArgSpec() = default; TFreeArgSpec(const TString& title, const TString& help = TString(), bool optional = false) : Title_(title) , Help_(help) , Optional_(optional) - { - } + { + } TString Title_; TString Help_; @@ -808,5 +808,5 @@ namespace NLastGetopt { Completer_ = std::move(completer); return *this; } - }; -} + }; +} diff --git a/library/cpp/getopt/small/last_getopt_opts.cpp b/library/cpp/getopt/small/last_getopt_opts.cpp index 03c432849f..bb04f03a7e 100644 --- a/library/cpp/getopt/small/last_getopt_opts.cpp +++ b/library/cpp/getopt/small/last_getopt_opts.cpp @@ -7,7 +7,7 @@ #include <util/stream/format.h> #include <util/charset/utf8.h> - + #include <stdlib.h> namespace NLastGetoptPrivate { @@ -19,212 +19,212 @@ namespace NLastGetoptPrivate { static TString data; return data; } -} +} -namespace NLastGetopt { +namespace NLastGetopt { static const TStringBuf SPad = " "; - void PrintVersionAndExit(const TOptsParser*) { + void PrintVersionAndExit(const TOptsParser*) { Cout << (NLastGetoptPrivate::VersionString() ? NLastGetoptPrivate::VersionString() : "program version: not linked with library/cpp/getopt") << Endl; - exit(NLastGetoptPrivate::VersionString().empty()); - } + exit(NLastGetoptPrivate::VersionString().empty()); + } void PrintShortVersionAndExit(const TString& appName) { Cout << appName << " version " << (NLastGetoptPrivate::ShortVersionString() ? NLastGetoptPrivate::ShortVersionString() : "not linked with library/cpp/getopt") << Endl; exit(NLastGetoptPrivate::ShortVersionString().empty()); } - // Like TString::Quote(), but does not quote digits-only string - static TString QuoteForHelp(const TString& str) { - if (str.empty()) - return str.Quote(); - for (size_t i = 0; i < str.size(); ++i) { - if (!isdigit(str[i])) - return str.Quote(); - } - return str; - } - - namespace NPrivate { - TString OptToString(char c) { - TStringStream ss; - ss << "-" << c; - return ss.Str(); - } - - TString OptToString(const TString& longOption) { - TStringStream ss; - ss << "--" << longOption; - return ss.Str(); - } - - TString OptToString(const TOpt* opt) { - return opt->ToShortString(); - } - } - - TOpts::TOpts(const TStringBuf& optstring) - : ArgPermutation_(DEFAULT_ARG_PERMUTATION) - , AllowSingleDashForLong_(false) - , AllowPlusForLong_(false) - , AllowUnknownCharOptions_(false) - , AllowUnknownLongOptions_(false) - , FreeArgsMin_(0) + // Like TString::Quote(), but does not quote digits-only string + static TString QuoteForHelp(const TString& str) { + if (str.empty()) + return str.Quote(); + for (size_t i = 0; i < str.size(); ++i) { + if (!isdigit(str[i])) + return str.Quote(); + } + return str; + } + + namespace NPrivate { + TString OptToString(char c) { + TStringStream ss; + ss << "-" << c; + return ss.Str(); + } + + TString OptToString(const TString& longOption) { + TStringStream ss; + ss << "--" << longOption; + return ss.Str(); + } + + TString OptToString(const TOpt* opt) { + return opt->ToShortString(); + } + } + + TOpts::TOpts(const TStringBuf& optstring) + : ArgPermutation_(DEFAULT_ARG_PERMUTATION) + , AllowSingleDashForLong_(false) + , AllowPlusForLong_(false) + , AllowUnknownCharOptions_(false) + , AllowUnknownLongOptions_(false) + , FreeArgsMin_(0) , FreeArgsMax_(UNLIMITED_ARGS) - { - if (!optstring.empty()) { - AddCharOptions(optstring); - } - AddVersionOption(0); - } - - void TOpts::AddCharOptions(const TStringBuf& optstring) { - size_t p = 0; - if (optstring[p] == '+') { - ArgPermutation_ = REQUIRE_ORDER; - ++p; - } else if (optstring[p] == '-') { - ArgPermutation_ = RETURN_IN_ORDER; - ++p; - } - - while (p < optstring.size()) { - char c = optstring[p]; - p++; - EHasArg ha = NO_ARGUMENT; - if (p < optstring.size() && optstring[p] == ':') { - ha = REQUIRED_ARGUMENT; - p++; + { + if (!optstring.empty()) { + AddCharOptions(optstring); + } + AddVersionOption(0); + } + + void TOpts::AddCharOptions(const TStringBuf& optstring) { + size_t p = 0; + if (optstring[p] == '+') { + ArgPermutation_ = REQUIRE_ORDER; + ++p; + } else if (optstring[p] == '-') { + ArgPermutation_ = RETURN_IN_ORDER; + ++p; + } + + while (p < optstring.size()) { + char c = optstring[p]; + p++; + EHasArg ha = NO_ARGUMENT; + if (p < optstring.size() && optstring[p] == ':') { + ha = REQUIRED_ARGUMENT; + p++; + } + if (p < optstring.size() && optstring[p] == ':') { + ha = OPTIONAL_ARGUMENT; + p++; + } + AddCharOption(c, ha); + } + } + + const TOpt* TOpts::FindLongOption(const TStringBuf& name) const { + for (const auto& Opt : Opts_) { + const TOpt* opt = Opt.Get(); + if (IsIn(opt->GetLongNames(), name)) + return opt; + } + return nullptr; + } + + TOpt* TOpts::FindLongOption(const TStringBuf& name) { + for (auto& Opt : Opts_) { + TOpt* opt = Opt.Get(); + if (IsIn(opt->GetLongNames(), name)) + return opt; + } + return nullptr; + } + + const TOpt* TOpts::FindCharOption(char c) const { + for (const auto& Opt : Opts_) { + const TOpt* opt = Opt.Get(); + if (IsIn(opt->GetShortNames(), c)) + return opt; + } + return nullptr; + } + + TOpt* TOpts::FindCharOption(char c) { + for (auto& Opt : Opts_) { + TOpt* opt = Opt.Get(); + if (IsIn(opt->GetShortNames(), c)) + return opt; + } + return nullptr; + } + + const TOpt& TOpts::GetCharOption(char c) const { + const TOpt* option = FindCharOption(c); + if (!option) + ythrow TException() << "unknown char option '" << c << "'"; + return *option; + } + + TOpt& TOpts::GetCharOption(char c) { + TOpt* option = FindCharOption(c); + if (!option) + ythrow TException() << "unknown char option '" << c << "'"; + return *option; + } + + const TOpt& TOpts::GetLongOption(const TStringBuf& name) const { + const TOpt* option = FindLongOption(name); + if (!option) + ythrow TException() << "unknown option " << name; + return *option; + } + + TOpt& TOpts::GetLongOption(const TStringBuf& name) { + TOpt* option = FindLongOption(name); + if (!option) + ythrow TException() << "unknown option " << name; + return *option; + } + + bool TOpts::HasAnyShortOption() const { + for (const auto& Opt : Opts_) { + const TOpt* opt = Opt.Get(); + if (!opt->GetShortNames().empty()) + return true; + } + return false; + } + + bool TOpts::HasAnyLongOption() const { + for (const auto& Opt : Opts_) { + TOpt* opt = Opt.Get(); + if (!opt->GetLongNames().empty()) + return true; + } + return false; + } + + void TOpts::Validate() const { + for (TOptsVector::const_iterator i = Opts_.begin(); i != Opts_.end(); ++i) { + TOpt* opt = i->Get(); + const TOpt::TShortNames& shortNames = opt->GetShortNames(); + for (auto c : shortNames) { + for (TOptsVector::const_iterator j = i + 1; j != Opts_.end(); ++j) { + TOpt* nextOpt = j->Get(); + if (nextOpt->CharIs(c)) + ythrow TConfException() << "option " + << NPrivate::OptToString(c) + << " is defined more than once"; + } } - if (p < optstring.size() && optstring[p] == ':') { - ha = OPTIONAL_ARGUMENT; - p++; + const TOpt::TLongNames& longNames = opt->GetLongNames(); + for (const auto& longName : longNames) { + for (TOptsVector::const_iterator j = i + 1; j != Opts_.end(); ++j) { + TOpt* nextOpt = j->Get(); + if (nextOpt->NameIs(longName)) + ythrow TConfException() << "option " + << NPrivate::OptToString(longName) + << " is defined more than once"; + } } - AddCharOption(c, ha); - } - } - - const TOpt* TOpts::FindLongOption(const TStringBuf& name) const { - for (const auto& Opt : Opts_) { - const TOpt* opt = Opt.Get(); - if (IsIn(opt->GetLongNames(), name)) - return opt; - } - return nullptr; - } - - TOpt* TOpts::FindLongOption(const TStringBuf& name) { - for (auto& Opt : Opts_) { - TOpt* opt = Opt.Get(); - if (IsIn(opt->GetLongNames(), name)) - return opt; - } - return nullptr; - } - - const TOpt* TOpts::FindCharOption(char c) const { - for (const auto& Opt : Opts_) { - const TOpt* opt = Opt.Get(); - if (IsIn(opt->GetShortNames(), c)) - return opt; - } - return nullptr; - } - - TOpt* TOpts::FindCharOption(char c) { - for (auto& Opt : Opts_) { - TOpt* opt = Opt.Get(); - if (IsIn(opt->GetShortNames(), c)) - return opt; - } - return nullptr; - } - - const TOpt& TOpts::GetCharOption(char c) const { - const TOpt* option = FindCharOption(c); - if (!option) - ythrow TException() << "unknown char option '" << c << "'"; - return *option; - } - - TOpt& TOpts::GetCharOption(char c) { - TOpt* option = FindCharOption(c); - if (!option) - ythrow TException() << "unknown char option '" << c << "'"; - return *option; - } - - const TOpt& TOpts::GetLongOption(const TStringBuf& name) const { - const TOpt* option = FindLongOption(name); - if (!option) - ythrow TException() << "unknown option " << name; - return *option; - } - - TOpt& TOpts::GetLongOption(const TStringBuf& name) { - TOpt* option = FindLongOption(name); - if (!option) - ythrow TException() << "unknown option " << name; - return *option; - } - - bool TOpts::HasAnyShortOption() const { - for (const auto& Opt : Opts_) { - const TOpt* opt = Opt.Get(); - if (!opt->GetShortNames().empty()) - return true; } - return false; - } - - bool TOpts::HasAnyLongOption() const { - for (const auto& Opt : Opts_) { - TOpt* opt = Opt.Get(); - if (!opt->GetLongNames().empty()) - return true; - } - return false; - } - - void TOpts::Validate() const { - for (TOptsVector::const_iterator i = Opts_.begin(); i != Opts_.end(); ++i) { - TOpt* opt = i->Get(); - const TOpt::TShortNames& shortNames = opt->GetShortNames(); - for (auto c : shortNames) { - for (TOptsVector::const_iterator j = i + 1; j != Opts_.end(); ++j) { - TOpt* nextOpt = j->Get(); - if (nextOpt->CharIs(c)) - ythrow TConfException() << "option " - << NPrivate::OptToString(c) - << " is defined more than once"; - } - } - const TOpt::TLongNames& longNames = opt->GetLongNames(); - for (const auto& longName : longNames) { - for (TOptsVector::const_iterator j = i + 1; j != Opts_.end(); ++j) { - TOpt* nextOpt = j->Get(); - if (nextOpt->NameIs(longName)) - ythrow TConfException() << "option " - << NPrivate::OptToString(longName) - << " is defined more than once"; - } - } - } - if (FreeArgsMax_ < FreeArgsMin_) { - ythrow TConfException() << "FreeArgsMax must be >= FreeArgsMin"; - } - if (!FreeArgSpecs_.empty() && FreeArgSpecs_.rbegin()->first >= FreeArgsMax_) { - ythrow TConfException() << "Described args count is greater than FreeArgsMax. Either increase FreeArgsMax or remove unreachable descriptions"; - } - } - - TOpt& TOpts::AddOption(const TOpt& option) { - if (option.GetShortNames().empty() && option.GetLongNames().empty()) - ythrow TConfException() << "bad option: no chars, no long names"; - Opts_.push_back(new TOpt(option)); - return *Opts_.back(); - } - + if (FreeArgsMax_ < FreeArgsMin_) { + ythrow TConfException() << "FreeArgsMax must be >= FreeArgsMin"; + } + if (!FreeArgSpecs_.empty() && FreeArgSpecs_.rbegin()->first >= FreeArgsMax_) { + ythrow TConfException() << "Described args count is greater than FreeArgsMax. Either increase FreeArgsMax or remove unreachable descriptions"; + } + } + + TOpt& TOpts::AddOption(const TOpt& option) { + if (option.GetShortNames().empty() && option.GetLongNames().empty()) + ythrow TConfException() << "bad option: no chars, no long names"; + Opts_.push_back(new TOpt(option)); + return *Opts_.back(); + } + TOpt& TOpts::AddCompletionOption(TString command, TString longName) { if (TOpt* o = FindLongOption(longName)) { return *o; @@ -252,19 +252,19 @@ namespace NLastGetopt { .IfPresentDisableCompletionFor(opt1); } - size_t TOpts::IndexOf(const TOpt* opt) const { - TOptsVector::const_iterator it = std::find(Opts_.begin(), Opts_.end(), opt); - if (it == Opts_.end()) - ythrow TException() << "unknown option"; - return it - Opts_.begin(); + size_t TOpts::IndexOf(const TOpt* opt) const { + TOptsVector::const_iterator it = std::find(Opts_.begin(), Opts_.end(), opt); + if (it == Opts_.end()) + ythrow TException() << "unknown option"; + return it - Opts_.begin(); } TStringBuf TOpts::GetFreeArgTitle(size_t pos) const { if (FreeArgSpecs_.contains(pos)) { return FreeArgSpecs_.at(pos).GetTitle(DefaultFreeArgTitle_); - } + } return DefaultFreeArgTitle_; - } + } void TOpts::SetFreeArgTitle(size_t pos, const TString& title, const TString& help, bool optional) { FreeArgSpecs_[pos] = TFreeArgSpec(title, help, optional); @@ -274,56 +274,56 @@ namespace NLastGetopt { return FreeArgSpecs_[pos]; } - static TString FormatOption(const TOpt* option, const NColorizer::TColors& colors) { - TStringStream result; - const TOpt::TShortNames& shorts = option->GetShortNames(); - const TOpt::TLongNames& longs = option->GetLongNames(); - - const size_t nopts = shorts.size() + longs.size(); - const bool multiple = 1 < nopts; - if (multiple) - result << '{'; - for (size_t i = 0; i < nopts; ++i) { - if (multiple && 0 != i) - result << '|'; - - if (i < shorts.size()) // short - result << colors.GreenColor() << '-' << shorts[i] << colors.OldColor(); - else - result << colors.GreenColor() << "--" << longs[i - shorts.size()] << colors.OldColor(); - } - if (multiple) - result << '}'; - - static const TString metavarDef("VAL"); - const TString& title = option->GetArgTitle(); + static TString FormatOption(const TOpt* option, const NColorizer::TColors& colors) { + TStringStream result; + const TOpt::TShortNames& shorts = option->GetShortNames(); + const TOpt::TLongNames& longs = option->GetLongNames(); + + const size_t nopts = shorts.size() + longs.size(); + const bool multiple = 1 < nopts; + if (multiple) + result << '{'; + for (size_t i = 0; i < nopts; ++i) { + if (multiple && 0 != i) + result << '|'; + + if (i < shorts.size()) // short + result << colors.GreenColor() << '-' << shorts[i] << colors.OldColor(); + else + result << colors.GreenColor() << "--" << longs[i - shorts.size()] << colors.OldColor(); + } + if (multiple) + result << '}'; + + static const TString metavarDef("VAL"); + const TString& title = option->GetArgTitle(); const TString& metavar = title.empty() ? metavarDef : title; - if (option->GetHasArg() == OPTIONAL_ARGUMENT) { - result << " [" << metavar; - if (option->HasOptionalValue()) - result << ':' << option->GetOptionalValue(); - result << ']'; - } else if (option->GetHasArg() == REQUIRED_ARGUMENT) - result << ' ' << metavar; + if (option->GetHasArg() == OPTIONAL_ARGUMENT) { + result << " [" << metavar; + if (option->HasOptionalValue()) + result << ':' << option->GetOptionalValue(); + result << ']'; + } else if (option->GetHasArg() == REQUIRED_ARGUMENT) + result << ' ' << metavar; else - Y_ASSERT(option->GetHasArg() == NO_ARGUMENT); + Y_ASSERT(option->GetHasArg() == NO_ARGUMENT); - return result.Str(); + return result.Str(); } - void TOpts::PrintCmdLine(const TStringBuf& program, IOutputStream& os, const NColorizer::TColors& colors) const { + void TOpts::PrintCmdLine(const TStringBuf& program, IOutputStream& os, const NColorizer::TColors& colors) const { os << colors.BoldColor() << "Usage" << colors.OldColor() << ": "; if (CustomUsage) { os << CustomUsage; } else { os << program << " "; } - if (CustomCmdLineDescr) { - os << CustomCmdLineDescr << Endl; - return; + if (CustomCmdLineDescr) { + os << CustomCmdLineDescr << Endl; + return; } - os << "[OPTIONS]"; + os << "[OPTIONS]"; ui32 numDescribedFlags = FreeArgSpecs_.empty() ? 0 : FreeArgSpecs_.rbegin()->first + 1; ui32 numArgsToShow = Max(FreeArgsMin_, FreeArgsMax_ == UNLIMITED_ARGS ? numDescribedFlags : FreeArgsMax_); @@ -342,92 +342,92 @@ namespace NLastGetopt { if (isOptional) os << "]"; - } - + } + if (FreeArgsMax_ == UNLIMITED_ARGS) { os << " [" << TrailingArgSpec_.GetTitle(DefaultFreeArgTitle_) << "]..."; } - os << Endl; - } + os << Endl; + } - void TOpts::PrintUsage(const TStringBuf& program, IOutputStream& osIn, const NColorizer::TColors& colors) const { - TStringStream os; + void TOpts::PrintUsage(const TStringBuf& program, IOutputStream& osIn, const NColorizer::TColors& colors) const { + TStringStream os; - if (!Title.empty()) - os << Title << "\n\n"; + if (!Title.empty()) + os << Title << "\n\n"; - PrintCmdLine(program, os, colors); + PrintCmdLine(program, os, colors); - TVector<TString> leftColumn(Opts_.size()); - TVector<size_t> leftColumnSizes(leftColumn.size()); + TVector<TString> leftColumn(Opts_.size()); + TVector<size_t> leftColumnSizes(leftColumn.size()); const size_t kMaxLeftWidth = 25; - size_t leftWidth = 0; - size_t requiredOptionsCount = 0; - NColorizer::TColors disabledColors(false); + size_t leftWidth = 0; + size_t requiredOptionsCount = 0; + NColorizer::TColors disabledColors(false); for (size_t i = 0; i < Opts_.size(); i++) { const TOpt* opt = Opts_[i].Get(); if (opt->IsHidden()) continue; - leftColumn[i] = FormatOption(opt, colors); + leftColumn[i] = FormatOption(opt, colors); size_t leftColumnSize = leftColumn[i].size(); if (colors.IsTTY()) { leftColumnSize -= NColorizer::TotalAnsiEscapeCodeLen(leftColumn[i]); } - leftColumnSizes[i] = leftColumnSize; + leftColumnSizes[i] = leftColumnSize; if (leftColumnSize <= kMaxLeftWidth) { leftWidth = Max(leftWidth, leftColumnSize); } - if (opt->IsRequired()) - requiredOptionsCount++; - } - - const TString leftPadding(leftWidth, ' '); - - for (size_t sectionId = 0; sectionId <= 1; sectionId++) { - bool requiredOptionsSection = (sectionId == 0); - - if (requiredOptionsSection) { - if (requiredOptionsCount == 0) - continue; - os << Endl << colors.BoldColor() << "Required parameters" << colors.OldColor() << ":" << Endl; + if (opt->IsRequired()) + requiredOptionsCount++; + } + + const TString leftPadding(leftWidth, ' '); + + for (size_t sectionId = 0; sectionId <= 1; sectionId++) { + bool requiredOptionsSection = (sectionId == 0); + + if (requiredOptionsSection) { + if (requiredOptionsCount == 0) + continue; + os << Endl << colors.BoldColor() << "Required parameters" << colors.OldColor() << ":" << Endl; } else { - if (requiredOptionsCount == Opts_.size()) - continue; - if (requiredOptionsCount == 0) - os << Endl << colors.BoldColor() << "Options" << colors.OldColor() << ":" << Endl; - else - os << Endl << colors.BoldColor() << "Optional parameters" << colors.OldColor() << ":" << Endl; // optional options would be a tautology + if (requiredOptionsCount == Opts_.size()) + continue; + if (requiredOptionsCount == 0) + os << Endl << colors.BoldColor() << "Options" << colors.OldColor() << ":" << Endl; + else + os << Endl << colors.BoldColor() << "Optional parameters" << colors.OldColor() << ":" << Endl; // optional options would be a tautology } - for (size_t i = 0; i < Opts_.size(); i++) { - const TOpt* opt = Opts_[i].Get(); - - if (opt->IsHidden()) - continue; - if (opt->IsRequired() != requiredOptionsSection) - continue; - - if (leftColumnSizes[i] > leftWidth && !opt->GetHelp().empty()) { + for (size_t i = 0; i < Opts_.size(); i++) { + const TOpt* opt = Opts_[i].Get(); + + if (opt->IsHidden()) + continue; + if (opt->IsRequired() != requiredOptionsSection) + continue; + + if (leftColumnSizes[i] > leftWidth && !opt->GetHelp().empty()) { os << SPad << leftColumn[i] << Endl << SPad << leftPadding << ' '; - } else { - os << SPad << leftColumn[i] << ' '; - if (leftColumnSizes[i] < leftWidth) + } else { + os << SPad << leftColumn[i] << ' '; + if (leftColumnSizes[i] < leftWidth) os << TStringBuf(leftPadding.data(), leftWidth - leftColumnSizes[i]); } TStringBuf help = opt->GetHelp(); while (help && isspace(help.back())) { help.Chop(1); - } + } size_t lastLineLength = 0; bool helpHasParagraphs = false; if (help) { os << Wrap(Wrap_, help, SPad + leftPadding + " ", &lastLineLength, &helpHasParagraphs); } - - if (opt->HasDefaultValue()) { + + if (opt->HasDefaultValue()) { auto quotedDef = QuoteForHelp(opt->GetDefaultValue()); if (helpHasParagraphs) { os << Endl << Endl << SPad << leftPadding << " "; @@ -445,9 +445,9 @@ namespace NLastGetopt { } else { os << "default: " << colors.CyanColor() << quotedDef << colors.OldColor(); } - } - - os << Endl; + } + + os << Endl; if (helpHasParagraphs) { os << Endl; @@ -455,7 +455,7 @@ namespace NLastGetopt { } } - PrintFreeArgsDesc(os, colors); + PrintFreeArgsDesc(os, colors); for (auto& [heading, text] : Sections) { os << Endl << colors.BoldColor() << heading << colors.OldColor() << ":" << Endl; @@ -463,40 +463,40 @@ namespace NLastGetopt { os << SPad << Wrap(Wrap_, text, SPad) << Endl; } - osIn << os.Str(); + osIn << os.Str(); } - void TOpts::PrintUsage(const TStringBuf& program, IOutputStream& os) const { - PrintUsage(program, os, NColorizer::AutoColors(os)); - } + void TOpts::PrintUsage(const TStringBuf& program, IOutputStream& os) const { + PrintUsage(program, os, NColorizer::AutoColors(os)); + } - void TOpts::PrintFreeArgsDesc(IOutputStream& os, const NColorizer::TColors& colors) const { - if (0 == FreeArgsMax_) - return; + void TOpts::PrintFreeArgsDesc(IOutputStream& os, const NColorizer::TColors& colors) const { + if (0 == FreeArgsMax_) + return; - size_t leftFreeWidth = 0; - for (size_t i = 0; i < FreeArgSpecs_.size(); ++i) { - leftFreeWidth = Max(leftFreeWidth, GetFreeArgTitle(i).size()); - } + size_t leftFreeWidth = 0; + for (size_t i = 0; i < FreeArgSpecs_.size(); ++i) { + leftFreeWidth = Max(leftFreeWidth, GetFreeArgTitle(i).size()); + } if (!TrailingArgSpec_.IsDefault()) { leftFreeWidth = Max(leftFreeWidth, TrailingArgSpec_.GetTitle(DefaultFreeArgTitle_).size()); - } + } - leftFreeWidth = Min(leftFreeWidth, size_t(30)); - os << Endl << colors.BoldColor() << "Free args" << colors.OldColor() << ":"; + leftFreeWidth = Min(leftFreeWidth, size_t(30)); + os << Endl << colors.BoldColor() << "Free args" << colors.OldColor() << ":"; - os << " min: " << colors.GreenColor() << FreeArgsMin_ << colors.OldColor() << ","; - os << " max: " << colors.GreenColor(); + os << " min: " << colors.GreenColor() << FreeArgsMin_ << colors.OldColor() << ","; + os << " max: " << colors.GreenColor(); if (FreeArgsMax_ != UNLIMITED_ARGS) { - os << FreeArgsMax_; - } else { - os << "unlimited"; - } + os << FreeArgsMax_; + } else { + os << "unlimited"; + } os << colors.OldColor() << Endl; - const size_t limit = FreeArgSpecs_.empty() ? 0 : FreeArgSpecs_.rbegin()->first; - for (size_t i = 0; i <= limit; ++i) { + const size_t limit = FreeArgSpecs_.empty() ? 0 : FreeArgSpecs_.rbegin()->first; + for (size_t i = 0; i <= limit; ++i) { if (!FreeArgSpecs_.contains(i)) { continue; } @@ -506,7 +506,7 @@ namespace NLastGetopt { os << SPad << colors.GreenColor() << RightPad(title, leftFreeWidth, ' ') << colors.OldColor() << SPad << help << Endl; } - } + } if (FreeArgsMax_ == UNLIMITED_ARGS) { auto title = TrailingArgSpec_.GetTitle(DefaultFreeArgTitle_); @@ -514,6 +514,6 @@ namespace NLastGetopt { os << SPad << colors.GreenColor() << RightPad(title, leftFreeWidth, ' ') << colors.OldColor() << SPad << help << Endl; } - } + } } } diff --git a/library/cpp/getopt/small/last_getopt_opts.h b/library/cpp/getopt/small/last_getopt_opts.h index 825b99c871..f7469ea313 100644 --- a/library/cpp/getopt/small/last_getopt_opts.h +++ b/library/cpp/getopt/small/last_getopt_opts.h @@ -7,14 +7,14 @@ #include <util/generic/map.h> namespace NLastGetopt { - enum EArgPermutation { - REQUIRE_ORDER, - PERMUTE, - RETURN_IN_ORDER, - DEFAULT_ARG_PERMUTATION = PERMUTE - }; - - /** + enum EArgPermutation { + REQUIRE_ORDER, + PERMUTE, + RETURN_IN_ORDER, + DEFAULT_ARG_PERMUTATION = PERMUTE + }; + + /** * NLastGetopt::TOpts is a storage of program options' parse rules. * It contains information about all options, free args, some parsing options * and rules about interaction between options. @@ -34,96 +34,96 @@ namespace NLastGetopt { * - Allowing unknown options * */ - class TOpts { - friend class TOptsParseResult; - friend class TOptsParser; + class TOpts { + friend class TOptsParseResult; + friend class TOptsParser; - public: + public: static constexpr const ui32 UNLIMITED_ARGS = Max<ui32>(); - typedef TVector<TSimpleSharedPtr<TOpt>> TOptsVector; - TOptsVector Opts_; // infomation about named (short and long) options + typedef TVector<TSimpleSharedPtr<TOpt>> TOptsVector; + TOptsVector Opts_; // infomation about named (short and long) options TVector<std::function<void(TStringBuf)>> ArgBindings_; - EArgPermutation ArgPermutation_ = DEFAULT_ARG_PERMUTATION; // determines how to parse positions of named and free options. See information below. - bool AllowSingleDashForLong_ = false; // - bool AllowPlusForLong_ = false; // using '+' instead '--' for long options + EArgPermutation ArgPermutation_ = DEFAULT_ARG_PERMUTATION; // determines how to parse positions of named and free options. See information below. + bool AllowSingleDashForLong_ = false; // + bool AllowPlusForLong_ = false; // using '+' instead '--' for long options - //Allows unknwon options: - bool AllowUnknownCharOptions_ = false; - bool AllowUnknownLongOptions_ = false; + //Allows unknwon options: + bool AllowUnknownCharOptions_ = false; + bool AllowUnknownLongOptions_ = false; ui32 Wrap_ = 80; - private: - ui32 FreeArgsMin_; // minimal number of free args - ui32 FreeArgsMax_; // maximal number of free args + private: + ui32 FreeArgsMin_; // minimal number of free args + ui32 FreeArgsMax_; // maximal number of free args TMap<ui32, TFreeArgSpec> FreeArgSpecs_; // mapping [free arg position] -> [free arg specification] TFreeArgSpec TrailingArgSpec_; // spec for the trailing argument (when arguments are unlimited) TString DefaultFreeArgTitle_ = "ARG"; // title that's used for free args without a title - TString Title; // title of the help string - TString CustomCmdLineDescr; // user defined help string + TString Title; // title of the help string + TString CustomCmdLineDescr; // user defined help string TString CustomUsage; // user defined usage string TVector<std::pair<TString, TString>> Sections; // additional help entries to print after usage - public: - /** + public: + /** * Constructs TOpts from string as in getopt(3) */ - TOpts(const TStringBuf& optstring = TStringBuf()); + TOpts(const TStringBuf& optstring = TStringBuf()); - /** + /** * Constructs TOpts from string as in getopt(3) and * additionally adds help option (for '?') and svn-verstion option (for 'V') */ - static TOpts Default(const TStringBuf& optstring = TStringBuf()) { - TOpts opts(optstring); - opts.AddHelpOption(); - opts.AddVersionOption(); - return opts; - } + static TOpts Default(const TStringBuf& optstring = TStringBuf()) { + TOpts opts(optstring); + opts.AddHelpOption(); + opts.AddVersionOption(); + return opts; + } - /** + /** * Checks correctness of options' descriptions. * Throws TConfException if validation failed. * Check consist of: * -not intersecting of names * -compability of settings, that responsable for freeArgs parsing */ - void Validate() const; + void Validate() const; - /** + /** * Search for the option with given long name * @param name long name for search * @return ptr on result (nullptr if not found) */ - const TOpt* FindLongOption(const TStringBuf& name) const; + const TOpt* FindLongOption(const TStringBuf& name) const; - /** + /** * Search for the option with given short name * @param c short name for search * @return ptr on result (nullptr if not found) */ - const TOpt* FindCharOption(char c) const; + const TOpt* FindCharOption(char c) const; - /** + /** * Search for the option with given long name * @param name long name for search * @return ptr on result (nullptr if not found) */ - TOpt* FindLongOption(const TStringBuf& name); + TOpt* FindLongOption(const TStringBuf& name); - /** + /** * Search for the option with given short name * @param c short name for search * @return ptr on result (nullptr if not found) */ - TOpt* FindCharOption(char c); + TOpt* FindCharOption(char c); - /** + /** * Search for the option with given name * @param name name for search * @return ptr on result (nullptr if not found) @@ -152,57 +152,57 @@ namespace NLastGetopt { * Sets title of the help string * @param title title to set */ - void SetTitle(const TString& title) { - Title = title; - } + void SetTitle(const TString& title) { + Title = title; + } - /** + /** * @return true if there is an option with given long name * * @param name long name for search */ - bool HasLongOption(const TString& name) const { - return FindLongOption(name) != nullptr; - } + bool HasLongOption(const TString& name) const { + return FindLongOption(name) != nullptr; + } - /** + /** * @return true if there is an option with given short name * * @param char short name for search */ - bool HasCharOption(char c) const { - return FindCharOption(c) != nullptr; - } + bool HasCharOption(char c) const { + return FindCharOption(c) != nullptr; + } - /** + /** * Search for the option with given long name * @param name long name for search * @return ref on result (throw exception if not found) */ - const TOpt& GetLongOption(const TStringBuf& name) const; + const TOpt& GetLongOption(const TStringBuf& name) const; - /** + /** * Search for the option with given long name * @param name long name for search * @return ref on result (throw exception if not found) */ - TOpt& GetLongOption(const TStringBuf& name); + TOpt& GetLongOption(const TStringBuf& name); - /** + /** * Search for the option with given short name * @param c short name for search * @return ref on result (throw exception if not found) */ - const TOpt& GetCharOption(char c) const; + const TOpt& GetCharOption(char c) const; - /** + /** * Search for the option with given short name * @param c short name for search * @return ref on result (throw exception if not found) */ - TOpt& GetCharOption(char c); + TOpt& GetCharOption(char c); - /** + /** * Search for the option with given name * @param name name for search * @return ref on result (throw exception if not found) @@ -230,21 +230,21 @@ namespace NLastGetopt { /** * @return true if short options exist */ - bool HasAnyShortOption() const; + bool HasAnyShortOption() const; - /** + /** * @return true if long options exist */ - bool HasAnyLongOption() const; + bool HasAnyLongOption() const; - /** + /** * Creates new [option description (TOpt)] as a copy of given one * @param option source * @return reference for created option */ - TOpt& AddOption(const TOpt& option); + TOpt& AddOption(const TOpt& option); - /** + /** * Creates new free argument handling * @param name name of free arg to show in help * @param target variable address to store parsing result into @@ -265,46 +265,46 @@ namespace NLastGetopt { * * @param optstring source */ - void AddCharOptions(const TStringBuf& optstring); + void AddCharOptions(const TStringBuf& optstring); - /** + /** * Creates new [option description (TOpt)] with given short name and given help string * * @param c short name * @param help help string * @return reference for created option */ - TOpt& AddCharOption(char c, const TString& help = "") { - return AddCharOption(c, DEFAULT_HAS_ARG, help); - } + TOpt& AddCharOption(char c, const TString& help = "") { + return AddCharOption(c, DEFAULT_HAS_ARG, help); + } - /** + /** * Creates new [option description (TOpt)] with given short name and given help string * * @param c short name * @param help help string * @return reference for created option */ - TOpt& AddCharOption(char c, EHasArg hasArg, const TString& help = "") { - TOpt option; - option.AddShortName(c); - option.Help(help); - option.HasArg(hasArg); - return AddOption(option); - } + TOpt& AddCharOption(char c, EHasArg hasArg, const TString& help = "") { + TOpt option; + option.AddShortName(c); + option.Help(help); + option.HasArg(hasArg); + return AddOption(option); + } - /** + /** * Creates new [option description (TOpt)] with given long name and given help string * * @param name long name * @param help help string * @return reference for created option */ - TOpt& AddLongOption(const TString& name, const TString& help = "") { - return AddLongOption(0, name, help); - } + TOpt& AddLongOption(const TString& name, const TString& help = "") { + return AddLongOption(0, name, help); + } - /** + /** * Creates new [option description (TOpt)] with given long and short names and given help string * * @param c short name @@ -312,54 +312,54 @@ namespace NLastGetopt { * @param help help string * @return reference for created option */ - TOpt& AddLongOption(char c, const TString& name, const TString& help = "") { - TOpt option; - if (c != 0) - option.AddShortName(c); - option.AddLongName(name); - option.Help(help); - return AddOption(option); - } + TOpt& AddLongOption(char c, const TString& name, const TString& help = "") { + TOpt option; + if (c != 0) + option.AddShortName(c); + option.AddLongName(name); + option.Help(help); + return AddOption(option); + } - /** + /** * Creates new [option description (TOpt)] for help printing, * adds appropriate handler for it * If "help" option already exist, will add given short name to it. * * @param c new short name for help option */ - TOpt& AddHelpOption(char c = '?') { - if (TOpt* o = FindLongOption("help")) { - if (!o->CharIs(c)) - o->AddShortName(c); - return *o; - } - return AddLongOption(c, "help", "print usage") - .HasArg(NO_ARGUMENT) + TOpt& AddHelpOption(char c = '?') { + if (TOpt* o = FindLongOption("help")) { + if (!o->CharIs(c)) + o->AddShortName(c); + return *o; + } + return AddLongOption(c, "help", "print usage") + .HasArg(NO_ARGUMENT) .IfPresentDisableCompletion() - .Handler(&PrintUsageAndExit); + .Handler(&PrintUsageAndExit); } - /** + /** * Creates new [option description (TOpt)] for svn-revision printing, * adds appropriate handler for it. * If "svnversion" option already exist, will add given short name to it. * * @param c new short name for "svnversion" option */ - TOpt& AddVersionOption(char c = 'V') { - if (TOpt* o = FindLongOption("svnrevision")) { - if (!o->CharIs(c)) - o->AddShortName(c); - return *o; - } - return AddLongOption(c, "svnrevision", "print svn version") - .HasArg(NO_ARGUMENT) + TOpt& AddVersionOption(char c = 'V') { + if (TOpt* o = FindLongOption("svnrevision")) { + if (!o->CharIs(c)) + o->AddShortName(c); + return *o; + } + return AddLongOption(c, "svnrevision", "print svn version") + .HasArg(NO_ARGUMENT) .IfPresentDisableCompletion() - .Handler(&PrintVersionAndExit); + .Handler(&PrintVersionAndExit); } - /** + /** * Creates new option for generating completion shell scripts. * * @param command name of command that should be completed (typically corresponds to the executable name). @@ -371,17 +371,17 @@ namespace NLastGetopt { * * @param c new short name for search/create */ - TOpt& CharOption(char c) { - const TOpt* opt = FindCharOption(c); - if (opt != nullptr) { - return const_cast<TOpt&>(*opt); - } else { - AddCharOption(c); - return const_cast<TOpt&>(GetCharOption(c)); - } + TOpt& CharOption(char c) { + const TOpt* opt = FindCharOption(c); + if (opt != nullptr) { + return const_cast<TOpt&>(*opt); + } else { + AddCharOption(c); + return const_cast<TOpt&>(GetCharOption(c)); + } } - /** + /** * Indicate that some options can't appear together. * * Note: this is not transitive. @@ -404,18 +404,18 @@ namespace NLastGetopt { * * @param opt pointer of option to search */ - size_t IndexOf(const TOpt* opt) const; + size_t IndexOf(const TOpt* opt) const; - /** + /** * Replace help string with given * * @param decr new help string */ - void SetCmdLineDescr(const TString& descr) { - CustomCmdLineDescr = descr; - } + void SetCmdLineDescr(const TString& descr) { + CustomCmdLineDescr = descr; + } - /** + /** * Replace usage string with given * * @param usage new usage string @@ -445,12 +445,12 @@ namespace NLastGetopt { * * @param min new value */ - void SetFreeArgsMin(size_t min) { - FreeArgsMin_ = ui32(min); - } + void SetFreeArgsMin(size_t min) { + FreeArgsMin_ = ui32(min); + } - /** + /** * Get current minimal number of free args */ ui32 GetFreeArgsMin() const { @@ -462,12 +462,12 @@ namespace NLastGetopt { * * @param max new value */ - void SetFreeArgsMax(size_t max) { - FreeArgsMax_ = ui32(max); + void SetFreeArgsMax(size_t max) { + FreeArgsMax_ = ui32(max); FreeArgsMax_ = Max<ui32>(FreeArgsMax_, ArgBindings_.size()); - } + } - /** + /** * Get current maximal number of free args */ ui32 GetFreeArgsMax() const { @@ -486,23 +486,23 @@ namespace NLastGetopt { * * @param count new value */ - void SetFreeArgsNum(size_t count) { - FreeArgsMin_ = ui32(count); - FreeArgsMax_ = ui32(count); - } + void SetFreeArgsNum(size_t count) { + FreeArgsMin_ = ui32(count); + FreeArgsMax_ = ui32(count); + } - /** + /** * Set minimal and maximal number of free args * * @param min new value for minimal * @param max new value for maximal */ - void SetFreeArgsNum(size_t min, size_t max) { - FreeArgsMin_ = ui32(min); - FreeArgsMax_ = ui32(max); - } + void SetFreeArgsNum(size_t min, size_t max) { + FreeArgsMin_ = ui32(min); + FreeArgsMax_ = ui32(max); + } - /** + /** * Set title and help string of free argument * * @param pos index of argument @@ -513,7 +513,7 @@ namespace NLastGetopt { */ void SetFreeArgTitle(size_t pos, const TString& title, const TString& help = TString(), bool optional = false); - /** + /** * Get free argument's spec for further modification. */ TFreeArgSpec& GetFreeArgSpec(size_t pos); @@ -556,7 +556,7 @@ namespace NLastGetopt { } /// @} - /** + /** * Get spec for the trailing argument. * * This spec is used to render the last repeated argument when max number of arguments is unlimited. @@ -575,11 +575,11 @@ namespace NLastGetopt { * * @param value new value of the option */ - void SetAllowSingleDashForLong(bool value) { - AllowSingleDashForLong_ = value; - } + void SetAllowSingleDashForLong(bool value) { + AllowSingleDashForLong_ = value; + } - /** + /** * Wrap help text at this number of characters. 0 to disable wrapping. */ void SetWrap(ui32 wrap = 80) { @@ -593,17 +593,17 @@ namespace NLastGetopt { * @param os destination stream * @param colors colorizer */ - void PrintUsage(const TStringBuf& program, IOutputStream& os, const NColorizer::TColors& colors) const; + void PrintUsage(const TStringBuf& program, IOutputStream& os, const NColorizer::TColors& colors) const; - /** + /** * Print usage string * * @param program prefix of result (path to the program) * @param os destination stream */ - void PrintUsage(const TStringBuf& program, IOutputStream& os = Cout) const; - - /** + void PrintUsage(const TStringBuf& program, IOutputStream& os = Cout) const; + + /** * Get list of options in order of definition. */ TVector<const TOpt*> GetOpts() const { @@ -615,29 +615,29 @@ namespace NLastGetopt { } private: - /** + /** * @return argument title of a free argument * * @param pos position of the argument */ TStringBuf GetFreeArgTitle(size_t pos) const; - /** + /** * Print usage helper * * @param program prefix of result (path to the program) * @param os destination stream * @param colors colorizer */ - void PrintCmdLine(const TStringBuf& program, IOutputStream& os, const NColorizer::TColors& colors) const; + void PrintCmdLine(const TStringBuf& program, IOutputStream& os, const NColorizer::TColors& colors) const; - /** + /** * Print usage helper * * @param os destination stream * @param colors colorizer */ - void PrintFreeArgsDesc(IOutputStream& os, const NColorizer::TColors& colors) const; - }; + void PrintFreeArgsDesc(IOutputStream& os, const NColorizer::TColors& colors) const; + }; -} +} diff --git a/library/cpp/getopt/small/last_getopt_parse_result.cpp b/library/cpp/getopt/small/last_getopt_parse_result.cpp index f4b5607a90..ec9e796cf3 100644 --- a/library/cpp/getopt/small/last_getopt_parse_result.cpp +++ b/library/cpp/getopt/small/last_getopt_parse_result.cpp @@ -1,131 +1,131 @@ #include "last_getopt_parse_result.h" namespace NLastGetopt { - const TOptParseResult* TOptsParseResult::FindParseResult(const TdVec& vec, const TOpt* opt) { - for (const auto& r : vec) { - if (r.OptPtr() == opt) - return &r; - } - return nullptr; - } - - const TOptParseResult* TOptsParseResult::FindOptParseResult(const TOpt* opt, bool includeDefault) const { - const TOptParseResult* r = FindParseResult(Opts_, opt); - if (nullptr == r && includeDefault) - r = FindParseResult(OptsDef_, opt); - return r; - } - - const TOptParseResult* TOptsParseResult::FindLongOptParseResult(const TString& name, bool includeDefault) const { - return FindOptParseResult(&Parser_->Opts_->GetLongOption(name), includeDefault); - } - - const TOptParseResult* TOptsParseResult::FindCharOptParseResult(char c, bool includeDefault) const { - return FindOptParseResult(&Parser_->Opts_->GetCharOption(c), includeDefault); - } - - bool TOptsParseResult::Has(const TOpt* opt, bool includeDefault) const { - Y_ASSERT(opt); - return FindOptParseResult(opt, includeDefault) != nullptr; - } - - bool TOptsParseResult::Has(const TString& name, bool includeDefault) const { - return FindLongOptParseResult(name, includeDefault) != nullptr; - } - - bool TOptsParseResult::Has(char c, bool includeDefault) const { - return FindCharOptParseResult(c, includeDefault) != nullptr; - } - - const char* TOptsParseResult::Get(const TOpt* opt, bool includeDefault) const { - Y_ASSERT(opt); - const TOptParseResult* r = FindOptParseResult(opt, includeDefault); - if (!r || r->Empty()) { - try { + const TOptParseResult* TOptsParseResult::FindParseResult(const TdVec& vec, const TOpt* opt) { + for (const auto& r : vec) { + if (r.OptPtr() == opt) + return &r; + } + return nullptr; + } + + const TOptParseResult* TOptsParseResult::FindOptParseResult(const TOpt* opt, bool includeDefault) const { + const TOptParseResult* r = FindParseResult(Opts_, opt); + if (nullptr == r && includeDefault) + r = FindParseResult(OptsDef_, opt); + return r; + } + + const TOptParseResult* TOptsParseResult::FindLongOptParseResult(const TString& name, bool includeDefault) const { + return FindOptParseResult(&Parser_->Opts_->GetLongOption(name), includeDefault); + } + + const TOptParseResult* TOptsParseResult::FindCharOptParseResult(char c, bool includeDefault) const { + return FindOptParseResult(&Parser_->Opts_->GetCharOption(c), includeDefault); + } + + bool TOptsParseResult::Has(const TOpt* opt, bool includeDefault) const { + Y_ASSERT(opt); + return FindOptParseResult(opt, includeDefault) != nullptr; + } + + bool TOptsParseResult::Has(const TString& name, bool includeDefault) const { + return FindLongOptParseResult(name, includeDefault) != nullptr; + } + + bool TOptsParseResult::Has(char c, bool includeDefault) const { + return FindCharOptParseResult(c, includeDefault) != nullptr; + } + + const char* TOptsParseResult::Get(const TOpt* opt, bool includeDefault) const { + Y_ASSERT(opt); + const TOptParseResult* r = FindOptParseResult(opt, includeDefault); + if (!r || r->Empty()) { + try { throw TUsageException() << "option " << opt->ToShortString() << " is unspecified"; - } catch (...) { - HandleError(); - // unreachable - throw; - } - } else { - return r->Back(); + } catch (...) { + HandleError(); + // unreachable + throw; + } + } else { + return r->Back(); + } + } + + const char* TOptsParseResult::GetOrElse(const TOpt* opt, const char* defaultValue) const { + Y_ASSERT(opt); + const TOptParseResult* r = FindOptParseResult(opt); + if (!r || r->Empty()) { + return defaultValue; + } else { + return r->Back(); } } - const char* TOptsParseResult::GetOrElse(const TOpt* opt, const char* defaultValue) const { - Y_ASSERT(opt); - const TOptParseResult* r = FindOptParseResult(opt); - if (!r || r->Empty()) { - return defaultValue; - } else { - return r->Back(); - } + const char* TOptsParseResult::Get(const TString& name, bool includeDefault) const { + return Get(&Parser_->Opts_->GetLongOption(name), includeDefault); } - const char* TOptsParseResult::Get(const TString& name, bool includeDefault) const { - return Get(&Parser_->Opts_->GetLongOption(name), includeDefault); - } + const char* TOptsParseResult::Get(char c, bool includeDefault) const { + return Get(&Parser_->Opts_->GetCharOption(c), includeDefault); + } - const char* TOptsParseResult::Get(char c, bool includeDefault) const { - return Get(&Parser_->Opts_->GetCharOption(c), includeDefault); - } + const char* TOptsParseResult::GetOrElse(const TString& name, const char* defaultValue) const { + if (!Has(name)) + return defaultValue; + return Get(name); + } - const char* TOptsParseResult::GetOrElse(const TString& name, const char* defaultValue) const { - if (!Has(name)) - return defaultValue; - return Get(name); - } + const char* TOptsParseResult::GetOrElse(char c, const char* defaultValue) const { + if (!Has(c)) + return defaultValue; + return Get(c); + } - const char* TOptsParseResult::GetOrElse(char c, const char* defaultValue) const { - if (!Has(c)) - return defaultValue; - return Get(c); - } + TOptParseResult& TOptsParseResult::OptParseResult() { + const TOpt* opt = Parser_->CurOpt(); + Y_ASSERT(opt); + TdVec& opts = Parser_->IsExplicit() ? Opts_ : OptsDef_; + if (Parser_->IsExplicit()) // default options won't appear twice + for (auto& it : opts) + if (it.OptPtr() == opt) + return it; + opts.push_back(TOptParseResult(opt)); + return opts.back(); + } - TOptParseResult& TOptsParseResult::OptParseResult() { - const TOpt* opt = Parser_->CurOpt(); - Y_ASSERT(opt); - TdVec& opts = Parser_->IsExplicit() ? Opts_ : OptsDef_; - if (Parser_->IsExplicit()) // default options won't appear twice - for (auto& it : opts) - if (it.OptPtr() == opt) - return it; - opts.push_back(TOptParseResult(opt)); - return opts.back(); - } - - TString TOptsParseResult::GetProgramName() const { - return Parser_->ProgramName_; - } + TString TOptsParseResult::GetProgramName() const { + return Parser_->ProgramName_; + } void TOptsParseResult::PrintUsage(IOutputStream& os) const { Parser_->Opts_->PrintUsage(Parser_->ProgramName_, os); } - size_t TOptsParseResult::GetFreeArgsPos() const { - return Parser_->Pos_; - } + size_t TOptsParseResult::GetFreeArgsPos() const { + return Parser_->Pos_; + } - TVector<TString> TOptsParseResult::GetFreeArgs() const { - TVector<TString> v; - for (size_t i = GetFreeArgsPos(); i < Parser_->Argc_; ++i) { - v.push_back(Parser_->Argv_[i]); - } - return v; - } + TVector<TString> TOptsParseResult::GetFreeArgs() const { + TVector<TString> v; + for (size_t i = GetFreeArgsPos(); i < Parser_->Argc_; ++i) { + v.push_back(Parser_->Argv_[i]); + } + return v; + } - size_t TOptsParseResult::GetFreeArgCount() const { - return Parser_->Argc_ - GetFreeArgsPos(); + size_t TOptsParseResult::GetFreeArgCount() const { + return Parser_->Argc_ - GetFreeArgsPos(); } - void TOptsParseResult::Init(const TOpts* options, int argc, const char** argv) { - try { - Parser_.Reset(new TOptsParser(options, argc, argv)); - while (Parser_->Next()) { - TOptParseResult& r = OptParseResult(); - r.AddValue(Parser_->CurValOrOpt().data()); - } + void TOptsParseResult::Init(const TOpts* options, int argc, const char** argv) { + try { + Parser_.Reset(new TOptsParser(options, argc, argv)); + while (Parser_->Next()) { + TOptParseResult& r = OptParseResult(); + r.AddValue(Parser_->CurValOrOpt().data()); + } Y_ENSURE(options); const auto freeArgs = GetFreeArgs(); @@ -136,25 +136,25 @@ namespace NLastGetopt { options->ArgBindings_[i](freeArgs[i]); } - } catch (...) { - HandleError(); + } catch (...) { + HandleError(); } } - void TOptsParseResult::HandleError() const { - Cerr << CurrentExceptionMessage() << Endl; - if (Parser_.Get()) { // parser initializing can fail (and we get here, see Init) - if (Parser_->Opts_->FindLongOption("help") != nullptr) { - Cerr << "Try '" << Parser_->ProgramName_ << " --help' for more information." << Endl; - } else { + void TOptsParseResult::HandleError() const { + Cerr << CurrentExceptionMessage() << Endl; + if (Parser_.Get()) { // parser initializing can fail (and we get here, see Init) + if (Parser_->Opts_->FindLongOption("help") != nullptr) { + Cerr << "Try '" << Parser_->ProgramName_ << " --help' for more information." << Endl; + } else { PrintUsage(); - } + } } - exit(1); + exit(1); } - void TOptsParseResultException::HandleError() const { - throw; - } + void TOptsParseResultException::HandleError() const { + throw; + } } diff --git a/library/cpp/getopt/small/last_getopt_parse_result.h b/library/cpp/getopt/small/last_getopt_parse_result.h index 1ab6f598c9..7f0da9c568 100644 --- a/library/cpp/getopt/small/last_getopt_parse_result.h +++ b/library/cpp/getopt/small/last_getopt_parse_result.h @@ -4,58 +4,58 @@ #include "last_getopt_parser.h" namespace NLastGetopt { - /** + /** * NLastGetopt::TOptParseResult contains all arguments for exactly one TOpt, * that have been fetched during parsing * * The class is a wraper over a vector of nil-terminated strings. */ - class TOptParseResult { - public: - typedef TVector<const char*> TValues; - - public: - TOptParseResult(const TOpt* opt = nullptr) - : Opt_(opt) - { - } - - public: - const TOpt& Opt() const { - return *Opt_; - } - const TOpt* OptPtr() const { - return Opt_; - } - const TValues& Values() const { - return Values_; - } - bool Empty() const { - return Values().empty(); - } - size_t Count() const { - return Values_.size(); - } - void AddValue(const char* val) { - if (nullptr != val) - Values_.push_back(val); - } - const char* DefVal(const char* def = nullptr) const { - return Opt().HasDefaultValue() ? Opt().GetDefaultValue().c_str() : def; - } - const char* Front(const char* def = nullptr) const { - return Empty() ? DefVal(def) : Values().front(); - } - const char* Back(const char* def = nullptr) const { - return Empty() ? DefVal(def) : Values().back(); - } - - private: - const TOpt* Opt_; - TValues Values_; - }; - - /** + class TOptParseResult { + public: + typedef TVector<const char*> TValues; + + public: + TOptParseResult(const TOpt* opt = nullptr) + : Opt_(opt) + { + } + + public: + const TOpt& Opt() const { + return *Opt_; + } + const TOpt* OptPtr() const { + return Opt_; + } + const TValues& Values() const { + return Values_; + } + bool Empty() const { + return Values().empty(); + } + size_t Count() const { + return Values_.size(); + } + void AddValue(const char* val) { + if (nullptr != val) + Values_.push_back(val); + } + const char* DefVal(const char* def = nullptr) const { + return Opt().HasDefaultValue() ? Opt().GetDefaultValue().c_str() : def; + } + const char* Front(const char* def = nullptr) const { + return Empty() ? DefVal(def) : Values().front(); + } + const char* Back(const char* def = nullptr) const { + return Empty() ? DefVal(def) : Values().back(); + } + + private: + const TOpt* Opt_; + TValues Values_; + }; + + /** * NLastGetopt::TOptsParseResult contains result of parsing argc,argv be parser. * * In most common case constructed by argc,argv pair and rules (TOpts). @@ -64,20 +64,20 @@ namespace NLastGetopt { * If error during parsing occures, the program aborts with exit code 1. * Note, that if PERMUTE mode is on, then data, pointed by argv can be changed. */ - class TOptsParseResult { - private: - THolder<TOptsParser> Parser_; //The instance of parser. + class TOptsParseResult { + private: + THolder<TOptsParser> Parser_; //The instance of parser. - // XXX: make argc, argv - typedef TVector<TOptParseResult> TdVec; + // XXX: make argc, argv + typedef TVector<TOptParseResult> TdVec; - TdVec Opts_; //Parsing result for all options, that have been explicitly defined in argc/argv - TdVec OptsDef_; //Parsing result for options, that have been defined by default values only + TdVec Opts_; //Parsing result for all options, that have been explicitly defined in argc/argv + TdVec OptsDef_; //Parsing result for options, that have been defined by default values only - private: - TOptParseResult& OptParseResult(); + private: + TOptParseResult& OptParseResult(); - /** + /** * Searchs for object in given container * * @param vec container @@ -85,49 +85,49 @@ namespace NLastGetopt { * * @retunr ptr on corresponding TOptParseResult */ - static const TOptParseResult* FindParseResult(const TdVec& vec, const TOpt* opt); + static const TOptParseResult* FindParseResult(const TdVec& vec, const TOpt* opt); - protected: - /** + protected: + /** * Performs parsing of comand line arguments. */ - void Init(const TOpts* options, int argc, const char** argv); + void Init(const TOpts* options, int argc, const char** argv); - TOptsParseResult() = default; + TOptsParseResult() = default; - public: - /** + public: + /** * The action in case of parser failure. * Allows to asjust behavior in derived classes. * By default prints error string and aborts the program */ - virtual void HandleError() const; + virtual void HandleError() const; - /** + /** * Constructs object by parsing arguments with given rules * * @param options ptr on parsing rules * @param argc * @param argv */ - TOptsParseResult(const TOpts* options, int argc, const char* argv[]) { - Init(options, argc, argv); - } + TOptsParseResult(const TOpts* options, int argc, const char* argv[]) { + Init(options, argc, argv); + } - /** + /** * Constructs object by parsing arguments with given rules * * @param options ptr on parsing rules * @param argc * @param argv */ - TOptsParseResult(const TOpts* options, int argc, char* argv[]) { - Init(options, argc, const_cast<const char**>(argv)); - } + TOptsParseResult(const TOpts* options, int argc, char* argv[]) { + Init(options, argc, const_cast<const char**>(argv)); + } - virtual ~TOptsParseResult() = default; + virtual ~TOptsParseResult() = default; - /** + /** * Search for TOptParseResult that corresponds to given option (TOpt) * * @param opt ptr on required object @@ -135,9 +135,9 @@ namespace NLastGetopt { * * @return ptr on result */ - const TOptParseResult* FindOptParseResult(const TOpt* opt, bool includeDefault = false) const; + const TOptParseResult* FindOptParseResult(const TOpt* opt, bool includeDefault = false) const; - /** + /** * Search for TOptParseResult that corresponds to given long name * * @param name long name of required object @@ -145,9 +145,9 @@ namespace NLastGetopt { * * @return ptr on result */ - const TOptParseResult* FindLongOptParseResult(const TString& name, bool includeDefault = false) const; + const TOptParseResult* FindLongOptParseResult(const TString& name, bool includeDefault = false) const; - /** + /** * Search for TOptParseResult that corresponds to given short name * * @param c short name of required object @@ -155,14 +155,14 @@ namespace NLastGetopt { * * @return ptr on result */ - const TOptParseResult* FindCharOptParseResult(char c, bool includeDefault = false) const; + const TOptParseResult* FindCharOptParseResult(char c, bool includeDefault = false) const; - /** + /** * @return argv[0] */ - TString GetProgramName() const; + TString GetProgramName() const; - /** + /** * Print usage string. */ void PrintUsage(IOutputStream& os = Cout) const; @@ -170,97 +170,97 @@ namespace NLastGetopt { /** * @return position in [premuted argv] of the first free argument */ - size_t GetFreeArgsPos() const; + size_t GetFreeArgsPos() const; - /** + /** * @return number of fetched free arguments */ - size_t GetFreeArgCount() const; + size_t GetFreeArgCount() const; - /** + /** * @return all fetched free arguments */ - TVector<TString> GetFreeArgs() const; + TVector<TString> GetFreeArgs() const; - /** + /** * @return true if given option exist in results of parsing * * @param opt ptr on required object * @param includeDefault search in results obtained from default values * */ - bool Has(const TOpt* opt, bool includeDefault = false) const; + bool Has(const TOpt* opt, bool includeDefault = false) const; - /** + /** * @return nil terminated string on the last fetched argument of givne option * * @param opt ptr on required object * @param includeDefault search in results obtained from default values */ - const char* Get(const TOpt* opt, bool includeDefault = true) const; + const char* Get(const TOpt* opt, bool includeDefault = true) const; - /** + /** * @return nil terminated string on the last fetched argument of givne option * if option haven't been fetched, given defaultValue will be returned * * @param opt ptr on required object * @param defaultValue */ - const char* GetOrElse(const TOpt* opt, const char* defaultValue) const; + const char* GetOrElse(const TOpt* opt, const char* defaultValue) const; - /** + /** * @return true if given option exist in results of parsing * * @param name long name of required object * @param includeDefault search in results obtained from default values * */ - bool Has(const TString& name, bool includeDefault = false) const; + bool Has(const TString& name, bool includeDefault = false) const; - /** + /** * @return nil terminated string on the last fetched argument of givne option * * @param name long name of required object * @param includeDefault search in results obtained from default values */ - const char* Get(const TString& name, bool includeDefault = true) const; + const char* Get(const TString& name, bool includeDefault = true) const; - /** + /** * @return nil terminated string on the last fetched argument of givne option * if option haven't been fetched, given defaultValue will be returned * * @param name long name of required object * @param defaultValue */ - const char* GetOrElse(const TString& name, const char* defaultValue) const; + const char* GetOrElse(const TString& name, const char* defaultValue) const; - /** + /** * @return true if given option exist in results of parsing * * @param c short name of required object * @param includeDefault search in results obtained from default values * */ - bool Has(char name, bool includeDefault = false) const; + bool Has(char name, bool includeDefault = false) const; - /** + /** * @return nil terminated string on the last fetched argument of givne option * * @param c short name of required object * @param includeDefault search in results obtained from default values */ - const char* Get(char name, bool includeDefault = true) const; + const char* Get(char name, bool includeDefault = true) const; - /** + /** * @return nil terminated string on the last fetched argument of givne option * if option haven't been fetched, given defaultValue will be returned * * @param c short name of required object * @param defaultValue */ - const char* GetOrElse(char name, const char* defaultValue) const; + const char* GetOrElse(char name, const char* defaultValue) const; - /** + /** * for givne option return parsed value of the last fetched argument * if option haven't been fetched, HandleError action is called * @@ -268,18 +268,18 @@ namespace NLastGetopt { * * @return FromString<T>(last feteched argument) */ - template <typename T, typename TKey> - T Get(const TKey opt) const { - const char* value = Get(opt); - try { - return NPrivate::OptFromString<T>(value, opt); - } catch (...) { - HandleError(); - throw; - } + template <typename T, typename TKey> + T Get(const TKey opt) const { + const char* value = Get(opt); + try { + return NPrivate::OptFromString<T>(value, opt); + } catch (...) { + HandleError(); + throw; + } } - /** + /** * for givne option return parsed value of the last fetched argument * if option haven't been fetched, given defaultValue will be returned * @@ -288,34 +288,34 @@ namespace NLastGetopt { * * @return FromString<T>(last feteched argument) */ - template <typename T, typename TKey> - T GetOrElse(const TKey opt, const T& defaultValue) const { - if (Has(opt)) - return Get<T>(opt); - else - return defaultValue; - } - }; - - /** + template <typename T, typename TKey> + T GetOrElse(const TKey opt, const T& defaultValue) const { + if (Has(opt)) + return Get<T>(opt); + else + return defaultValue; + } + }; + + /** * NLastGetopt::TOptsParseResultException contains result of parsing argc,argv be parser. * * Unlike TOptsParseResult, if error during parsing occures, an exception is thrown. * */ - class TOptsParseResultException: public TOptsParseResult { - public: - TOptsParseResultException(const TOpts* options, int argc, const char* argv[]) { - Init(options, argc, argv); - } - TOptsParseResultException(const TOpts* options, int argc, char* argv[]) { - Init(options, argc, const_cast<const char**>(argv)); - } - virtual ~TOptsParseResultException() = default; - void HandleError() const override; - - protected: - TOptsParseResultException() = default; - }; + class TOptsParseResultException: public TOptsParseResult { + public: + TOptsParseResultException(const TOpts* options, int argc, const char* argv[]) { + Init(options, argc, argv); + } + TOptsParseResultException(const TOpts* options, int argc, char* argv[]) { + Init(options, argc, const_cast<const char**>(argv)); + } + virtual ~TOptsParseResultException() = default; + void HandleError() const override; + + protected: + TOptsParseResultException() = default; + }; } diff --git a/library/cpp/getopt/small/last_getopt_parser.cpp b/library/cpp/getopt/small/last_getopt_parser.cpp index 7668b12a03..84d4bd2e33 100644 --- a/library/cpp/getopt/small/last_getopt_parser.cpp +++ b/library/cpp/getopt/small/last_getopt_parser.cpp @@ -5,385 +5,385 @@ #include <util/string/escape.h> namespace NLastGetopt { - void TOptsParser::Init(const TOpts* opts, int argc, const char* argv[]) { - opts->Validate(); + void TOptsParser::Init(const TOpts* opts, int argc, const char* argv[]) { + opts->Validate(); - Opts_ = opts; + Opts_ = opts; - if (argc < 1) + if (argc < 1) throw TUsageException() << "argv must have at least one argument"; - Argc_ = argc; - Argv_ = argv; - - ProgramName_ = argv[0]; - - Pos_ = 1; - Sop_ = 0; - CurrentOpt_ = nullptr; - CurrentValue_ = nullptr; - GotMinusMinus_ = false; - Stopped_ = false; - OptsSeen_.clear(); - OptsDefault_.clear(); - } - - void TOptsParser::Init(const TOpts* opts, int argc, char* argv[]) { - Init(opts, argc, const_cast<const char**>(argv)); - } - - void TOptsParser::Swap(TOptsParser& that) { - DoSwap(Opts_, that.Opts_); - DoSwap(Argc_, that.Argc_); - DoSwap(Argv_, that.Argv_); - DoSwap(TempCurrentOpt_, that.TempCurrentOpt_); - DoSwap(ProgramName_, that.ProgramName_); - DoSwap(Pos_, that.Pos_); - DoSwap(Sop_, that.Sop_); - DoSwap(Stopped_, that.Stopped_); - DoSwap(CurrentOpt_, that.CurrentOpt_); - DoSwap(CurrentValue_, that.CurrentValue_); - DoSwap(GotMinusMinus_, that.GotMinusMinus_); - DoSwap(OptsSeen_, that.OptsSeen_); - } - - bool TOptsParser::Commit(const TOpt* currentOpt, const TStringBuf& currentValue, size_t pos, size_t sop) { - Pos_ = pos; - Sop_ = sop; - CurrentOpt_ = currentOpt; - CurrentValue_ = currentValue; - if (nullptr != currentOpt) - OptsSeen_.insert(currentOpt); - return true; - } - - bool TOptsParser::CommitEndOfOptions(size_t pos) { - Pos_ = pos; - Sop_ = 0; - Y_ASSERT(!CurOpt()); - Y_ASSERT(!CurVal()); - - Y_ASSERT(!Stopped_); - - if (Opts_->FreeArgsMin_ == Opts_->FreeArgsMax_ && Argc_ - Pos_ != Opts_->FreeArgsMin_) + Argc_ = argc; + Argv_ = argv; + + ProgramName_ = argv[0]; + + Pos_ = 1; + Sop_ = 0; + CurrentOpt_ = nullptr; + CurrentValue_ = nullptr; + GotMinusMinus_ = false; + Stopped_ = false; + OptsSeen_.clear(); + OptsDefault_.clear(); + } + + void TOptsParser::Init(const TOpts* opts, int argc, char* argv[]) { + Init(opts, argc, const_cast<const char**>(argv)); + } + + void TOptsParser::Swap(TOptsParser& that) { + DoSwap(Opts_, that.Opts_); + DoSwap(Argc_, that.Argc_); + DoSwap(Argv_, that.Argv_); + DoSwap(TempCurrentOpt_, that.TempCurrentOpt_); + DoSwap(ProgramName_, that.ProgramName_); + DoSwap(Pos_, that.Pos_); + DoSwap(Sop_, that.Sop_); + DoSwap(Stopped_, that.Stopped_); + DoSwap(CurrentOpt_, that.CurrentOpt_); + DoSwap(CurrentValue_, that.CurrentValue_); + DoSwap(GotMinusMinus_, that.GotMinusMinus_); + DoSwap(OptsSeen_, that.OptsSeen_); + } + + bool TOptsParser::Commit(const TOpt* currentOpt, const TStringBuf& currentValue, size_t pos, size_t sop) { + Pos_ = pos; + Sop_ = sop; + CurrentOpt_ = currentOpt; + CurrentValue_ = currentValue; + if (nullptr != currentOpt) + OptsSeen_.insert(currentOpt); + return true; + } + + bool TOptsParser::CommitEndOfOptions(size_t pos) { + Pos_ = pos; + Sop_ = 0; + Y_ASSERT(!CurOpt()); + Y_ASSERT(!CurVal()); + + Y_ASSERT(!Stopped_); + + if (Opts_->FreeArgsMin_ == Opts_->FreeArgsMax_ && Argc_ - Pos_ != Opts_->FreeArgsMin_) throw TUsageException() << "required exactly " << Opts_->FreeArgsMin_ << " free args"; - else if (Argc_ - Pos_ < Opts_->FreeArgsMin_) + else if (Argc_ - Pos_ < Opts_->FreeArgsMin_) throw TUsageException() << "required at least " << Opts_->FreeArgsMin_ << " free args"; - else if (Argc_ - Pos_ > Opts_->FreeArgsMax_) + else if (Argc_ - Pos_ > Opts_->FreeArgsMax_) throw TUsageException() << "required at most " << Opts_->FreeArgsMax_ << " free args"; - return false; - } + return false; + } - bool TOptsParser::ParseUnknownShortOptWithinArg(size_t pos, size_t sop) { - Y_ASSERT(pos < Argc_); - const TStringBuf arg(Argv_[pos]); - Y_ASSERT(sop > 0); - Y_ASSERT(sop < arg.length()); - Y_ASSERT(EIO_NONE != IsOpt(arg)); + bool TOptsParser::ParseUnknownShortOptWithinArg(size_t pos, size_t sop) { + Y_ASSERT(pos < Argc_); + const TStringBuf arg(Argv_[pos]); + Y_ASSERT(sop > 0); + Y_ASSERT(sop < arg.length()); + Y_ASSERT(EIO_NONE != IsOpt(arg)); - if (!Opts_->AllowUnknownCharOptions_) + if (!Opts_->AllowUnknownCharOptions_) throw TUsageException() << "unknown option '" << EscapeC(arg[sop]) - << "' in '" << arg << "'"; + << "' in '" << arg << "'"; - TempCurrentOpt_.Reset(new TOpt); - TempCurrentOpt_->AddShortName(arg[sop]); + TempCurrentOpt_.Reset(new TOpt); + TempCurrentOpt_->AddShortName(arg[sop]); - sop += 1; + sop += 1; - // mimic behavior of Opt: unknown option has arg only if char is last within arg - if (sop < arg.length()) { - return Commit(TempCurrentOpt_.Get(), nullptr, pos, sop); - } + // mimic behavior of Opt: unknown option has arg only if char is last within arg + if (sop < arg.length()) { + return Commit(TempCurrentOpt_.Get(), nullptr, pos, sop); + } - pos += 1; - sop = 0; - if (pos == Argc_ || EIO_NONE != IsOpt(Argv_[pos])) { - return Commit(TempCurrentOpt_.Get(), nullptr, pos, 0); - } + pos += 1; + sop = 0; + if (pos == Argc_ || EIO_NONE != IsOpt(Argv_[pos])) { + return Commit(TempCurrentOpt_.Get(), nullptr, pos, 0); + } - return Commit(TempCurrentOpt_.Get(), Argv_[pos], pos + 1, 0); + return Commit(TempCurrentOpt_.Get(), Argv_[pos], pos + 1, 0); } - bool TOptsParser::ParseShortOptWithinArg(size_t pos, size_t sop) { - Y_ASSERT(pos < Argc_); - const TStringBuf arg(Argv_[pos]); - Y_ASSERT(sop > 0); - Y_ASSERT(sop < arg.length()); - Y_ASSERT(EIO_NONE != IsOpt(arg)); - - size_t p = sop; - char c = arg[p]; - const TOpt* opt = Opts_->FindCharOption(c); - if (!opt) - return ParseUnknownShortOptWithinArg(pos, sop); - p += 1; - if (p == arg.length()) { - return ParseOptParam(opt, pos + 1); - } - if (opt->GetHasArg() == NO_ARGUMENT) { - return Commit(opt, nullptr, pos, p); - } + bool TOptsParser::ParseShortOptWithinArg(size_t pos, size_t sop) { + Y_ASSERT(pos < Argc_); + const TStringBuf arg(Argv_[pos]); + Y_ASSERT(sop > 0); + Y_ASSERT(sop < arg.length()); + Y_ASSERT(EIO_NONE != IsOpt(arg)); + + size_t p = sop; + char c = arg[p]; + const TOpt* opt = Opts_->FindCharOption(c); + if (!opt) + return ParseUnknownShortOptWithinArg(pos, sop); + p += 1; + if (p == arg.length()) { + return ParseOptParam(opt, pos + 1); + } + if (opt->GetHasArg() == NO_ARGUMENT) { + return Commit(opt, nullptr, pos, p); + } return Commit(opt, arg.SubStr(p), pos + 1, 0); + } + + bool TOptsParser::ParseShortOptArg(size_t pos) { + Y_ASSERT(pos < Argc_); + const TStringBuf arg(Argv_[pos]); + Y_ASSERT(EIO_NONE != IsOpt(arg)); + Y_ASSERT(!arg.StartsWith("--")); + return ParseShortOptWithinArg(pos, 1); } - bool TOptsParser::ParseShortOptArg(size_t pos) { - Y_ASSERT(pos < Argc_); - const TStringBuf arg(Argv_[pos]); - Y_ASSERT(EIO_NONE != IsOpt(arg)); - Y_ASSERT(!arg.StartsWith("--")); - return ParseShortOptWithinArg(pos, 1); - } - - bool TOptsParser::ParseOptArg(size_t pos) { - Y_ASSERT(pos < Argc_); - TStringBuf arg(Argv_[pos]); - const EIsOpt eio = IsOpt(arg); - Y_ASSERT(EIO_NONE != eio); - if (EIO_DDASH == eio || EIO_PLUS == eio || (Opts_->AllowSingleDashForLong_ || !Opts_->HasAnyShortOption())) { - // long option - bool singleCharPrefix = EIO_DDASH != eio; - arg.Skip(singleCharPrefix ? 1 : 2); - TStringBuf optionName = arg.NextTok('='); - const TOpt* option = Opts_->FindLongOption(optionName); - if (!option) { - if (singleCharPrefix && !arg.IsInited()) { - return ParseShortOptArg(pos); - } else if (Opts_->AllowUnknownLongOptions_) { - return false; - } else { + bool TOptsParser::ParseOptArg(size_t pos) { + Y_ASSERT(pos < Argc_); + TStringBuf arg(Argv_[pos]); + const EIsOpt eio = IsOpt(arg); + Y_ASSERT(EIO_NONE != eio); + if (EIO_DDASH == eio || EIO_PLUS == eio || (Opts_->AllowSingleDashForLong_ || !Opts_->HasAnyShortOption())) { + // long option + bool singleCharPrefix = EIO_DDASH != eio; + arg.Skip(singleCharPrefix ? 1 : 2); + TStringBuf optionName = arg.NextTok('='); + const TOpt* option = Opts_->FindLongOption(optionName); + if (!option) { + if (singleCharPrefix && !arg.IsInited()) { + return ParseShortOptArg(pos); + } else if (Opts_->AllowUnknownLongOptions_) { + return false; + } else { throw TUsageException() << "unknown option '" << optionName - << "' in '" << Argv_[pos] << "'"; - } + << "' in '" << Argv_[pos] << "'"; + } } - if (arg.IsInited()) { - if (option->GetHasArg() == NO_ARGUMENT) + if (arg.IsInited()) { + if (option->GetHasArg() == NO_ARGUMENT) throw TUsageException() << "option " << optionName << " must have no arg"; - return Commit(option, arg, pos + 1, 0); - } - ++pos; - return ParseOptParam(option, pos); - } else { - return ParseShortOptArg(pos); + return Commit(option, arg, pos + 1, 0); + } + ++pos; + return ParseOptParam(option, pos); + } else { + return ParseShortOptArg(pos); } } - bool TOptsParser::ParseOptParam(const TOpt* opt, size_t pos) { - Y_ASSERT(opt); - if (opt->GetHasArg() == NO_ARGUMENT) { - return Commit(opt, nullptr, pos, 0); - } - if (pos == Argc_) { - if (opt->GetHasArg() == REQUIRED_ARGUMENT) + bool TOptsParser::ParseOptParam(const TOpt* opt, size_t pos) { + Y_ASSERT(opt); + if (opt->GetHasArg() == NO_ARGUMENT) { + return Commit(opt, nullptr, pos, 0); + } + if (pos == Argc_) { + if (opt->GetHasArg() == REQUIRED_ARGUMENT) throw TUsageException() << "option " << opt->ToShortString() << " must have arg"; - return Commit(opt, nullptr, pos, 0); - } - const TStringBuf arg(Argv_[pos]); - if (!arg.StartsWith('-') || opt->GetHasArg() == REQUIRED_ARGUMENT) { - return Commit(opt, arg, pos + 1, 0); - } + return Commit(opt, nullptr, pos, 0); + } + const TStringBuf arg(Argv_[pos]); + if (!arg.StartsWith('-') || opt->GetHasArg() == REQUIRED_ARGUMENT) { + return Commit(opt, arg, pos + 1, 0); + } return Commit(opt, nullptr, pos, 0); } - TOptsParser::EIsOpt TOptsParser::IsOpt(const TStringBuf& arg) const { - EIsOpt eio = EIO_NONE; - if (1 < arg.length()) { - switch (arg[0]) { - default: - break; - case '-': - if ('-' != arg[1]) - eio = EIO_SDASH; - else if (2 < arg.length()) - eio = EIO_DDASH; - break; - case '+': - if (Opts_->AllowPlusForLong_) - eio = EIO_PLUS; - break; - } + TOptsParser::EIsOpt TOptsParser::IsOpt(const TStringBuf& arg) const { + EIsOpt eio = EIO_NONE; + if (1 < arg.length()) { + switch (arg[0]) { + default: + break; + case '-': + if ('-' != arg[1]) + eio = EIO_SDASH; + else if (2 < arg.length()) + eio = EIO_DDASH; + break; + case '+': + if (Opts_->AllowPlusForLong_) + eio = EIO_PLUS; + break; + } } - return eio; + return eio; } - static void memrotate(void* ptr, size_t size, size_t shift) { - TTempBuf buf(shift); - memcpy(buf.Data(), (char*)ptr + size - shift, shift); - memmove((char*)ptr + shift, ptr, size - shift); - memcpy(ptr, buf.Data(), shift); - } + static void memrotate(void* ptr, size_t size, size_t shift) { + TTempBuf buf(shift); + memcpy(buf.Data(), (char*)ptr + size - shift, shift); + memmove((char*)ptr + shift, ptr, size - shift); + memcpy(ptr, buf.Data(), shift); + } - bool TOptsParser::ParseWithPermutation() { - Y_ASSERT(Sop_ == 0); - Y_ASSERT(Opts_->ArgPermutation_ == PERMUTE); + bool TOptsParser::ParseWithPermutation() { + Y_ASSERT(Sop_ == 0); + Y_ASSERT(Opts_->ArgPermutation_ == PERMUTE); - const size_t p0 = Pos_; + const size_t p0 = Pos_; - size_t pc = Pos_; + size_t pc = Pos_; - for (; pc < Argc_ && EIO_NONE == IsOpt(Argv_[pc]); ++pc) { - // count non-args - } + for (; pc < Argc_ && EIO_NONE == IsOpt(Argv_[pc]); ++pc) { + // count non-args + } - if (pc == Argc_) { - return CommitEndOfOptions(Pos_); - } + if (pc == Argc_) { + return CommitEndOfOptions(Pos_); + } - Pos_ = pc; + Pos_ = pc; - bool r = ParseOptArg(Pos_); + bool r = ParseOptArg(Pos_); Y_ASSERT(r); - while (Pos_ == pc) { - Y_ASSERT(Sop_ > 0); - r = ParseShortOptWithinArg(Pos_, Sop_); - Y_ASSERT(r); - } - - size_t p2 = Pos_; - - Y_ASSERT(p2 - pc >= 1); - Y_ASSERT(p2 - pc <= 2); - - memrotate(Argv_ + p0, (p2 - p0) * sizeof(void*), (p2 - pc) * sizeof(void*)); - - bool r2 = ParseOptArg(p0); - Y_ASSERT(r2); - return r2; - } - - bool TOptsParser::DoNext() { - Y_ASSERT(Pos_ <= Argc_); - - if (Pos_ == Argc_) - return CommitEndOfOptions(Pos_); - - if (GotMinusMinus_ && Opts_->ArgPermutation_ == RETURN_IN_ORDER) { - Y_ASSERT(Sop_ == 0); - return Commit(nullptr, Argv_[Pos_], Pos_ + 1, 0); - } - - if (Sop_ > 0) - return ParseShortOptWithinArg(Pos_, Sop_); - - size_t pos = Pos_; - const TStringBuf arg(Argv_[pos]); - if (EIO_NONE != IsOpt(arg)) { - return ParseOptArg(pos); - } else if (arg == "--") { - if (Opts_->ArgPermutation_ == RETURN_IN_ORDER) { - pos += 1; - if (pos == Argc_) - return CommitEndOfOptions(pos); - GotMinusMinus_ = true; - return Commit(nullptr, Argv_[pos], pos + 1, 0); - } else { - return CommitEndOfOptions(pos + 1); - } - } else if (Opts_->ArgPermutation_ == RETURN_IN_ORDER) { - return Commit(nullptr, arg, pos + 1, 0); - } else if (Opts_->ArgPermutation_ == REQUIRE_ORDER) { - return CommitEndOfOptions(Pos_); + while (Pos_ == pc) { + Y_ASSERT(Sop_ > 0); + r = ParseShortOptWithinArg(Pos_, Sop_); + Y_ASSERT(r); + } + + size_t p2 = Pos_; + + Y_ASSERT(p2 - pc >= 1); + Y_ASSERT(p2 - pc <= 2); + + memrotate(Argv_ + p0, (p2 - p0) * sizeof(void*), (p2 - pc) * sizeof(void*)); + + bool r2 = ParseOptArg(p0); + Y_ASSERT(r2); + return r2; + } + + bool TOptsParser::DoNext() { + Y_ASSERT(Pos_ <= Argc_); + + if (Pos_ == Argc_) + return CommitEndOfOptions(Pos_); + + if (GotMinusMinus_ && Opts_->ArgPermutation_ == RETURN_IN_ORDER) { + Y_ASSERT(Sop_ == 0); + return Commit(nullptr, Argv_[Pos_], Pos_ + 1, 0); + } + + if (Sop_ > 0) + return ParseShortOptWithinArg(Pos_, Sop_); + + size_t pos = Pos_; + const TStringBuf arg(Argv_[pos]); + if (EIO_NONE != IsOpt(arg)) { + return ParseOptArg(pos); + } else if (arg == "--") { + if (Opts_->ArgPermutation_ == RETURN_IN_ORDER) { + pos += 1; + if (pos == Argc_) + return CommitEndOfOptions(pos); + GotMinusMinus_ = true; + return Commit(nullptr, Argv_[pos], pos + 1, 0); + } else { + return CommitEndOfOptions(pos + 1); + } + } else if (Opts_->ArgPermutation_ == RETURN_IN_ORDER) { + return Commit(nullptr, arg, pos + 1, 0); + } else if (Opts_->ArgPermutation_ == REQUIRE_ORDER) { + return CommitEndOfOptions(Pos_); } else { - return ParseWithPermutation(); + return ParseWithPermutation(); } } - bool TOptsParser::Next() { - bool r = false; - - if (OptsDefault_.empty()) { - CurrentOpt_ = nullptr; - TempCurrentOpt_.Destroy(); + bool TOptsParser::Next() { + bool r = false; - CurrentValue_ = nullptr; + if (OptsDefault_.empty()) { + CurrentOpt_ = nullptr; + TempCurrentOpt_.Destroy(); - if (Stopped_) - return false; + CurrentValue_ = nullptr; - TOptsParser copy = *this; + if (Stopped_) + return false; - r = copy.DoNext(); + TOptsParser copy = *this; - Swap(copy); + r = copy.DoNext(); - if (!r) { - Stopped_ = true; - // we are done; check for missing options - Finish(); - } - } + Swap(copy); - if (!r && !OptsDefault_.empty()) { - CurrentOpt_ = OptsDefault_.front(); - CurrentValue_ = CurrentOpt_->GetDefaultValue(); - OptsDefault_.pop_front(); - r = true; + if (!r) { + Stopped_ = true; + // we are done; check for missing options + Finish(); + } } - if (r) { - if (CurOpt()) - CurOpt()->FireHandlers(this); - } - - return r; + if (!r && !OptsDefault_.empty()) { + CurrentOpt_ = OptsDefault_.front(); + CurrentValue_ = CurrentOpt_->GetDefaultValue(); + OptsDefault_.pop_front(); + r = true; + } + + if (r) { + if (CurOpt()) + CurOpt()->FireHandlers(this); + } + + return r; } - void TOptsParser::Finish() { - const TOpts::TOptsVector& optvec = Opts_->Opts_; - if (optvec.size() == OptsSeen_.size()) - return; + void TOptsParser::Finish() { + const TOpts::TOptsVector& optvec = Opts_->Opts_; + if (optvec.size() == OptsSeen_.size()) + return; - TVector<TString> missingLong; - TVector<char> missingShort; + TVector<TString> missingLong; + TVector<char> missingShort; - TOpts::TOptsVector::const_iterator it; - for (it = optvec.begin(); it != optvec.end(); ++it) { - const TOpt* opt = (*it).Get(); - if (nullptr == opt) - continue; + TOpts::TOptsVector::const_iterator it; + for (it = optvec.begin(); it != optvec.end(); ++it) { + const TOpt* opt = (*it).Get(); + if (nullptr == opt) + continue; if (OptsSeen_.contains(opt)) - continue; - - if (opt->IsRequired()) { - const TOpt::TLongNames& optnames = opt->GetLongNames(); - if (!optnames.empty()) - missingLong.push_back(optnames[0]); - else { - const char ch = opt->GetCharOr0(); - if (0 != ch) - missingShort.push_back(ch); - } - continue; - } - - if (opt->HasDefaultValue()) - OptsDefault_.push_back(opt); + continue; + + if (opt->IsRequired()) { + const TOpt::TLongNames& optnames = opt->GetLongNames(); + if (!optnames.empty()) + missingLong.push_back(optnames[0]); + else { + const char ch = opt->GetCharOr0(); + if (0 != ch) + missingShort.push_back(ch); + } + continue; + } + + if (opt->HasDefaultValue()) + OptsDefault_.push_back(opt); } - // also indicates subsequent options, if any, haven't been seen actually - OptsSeen_.clear(); - - const size_t nmissing = missingLong.size() + missingShort.size(); - if (0 == nmissing) - return; - - TUsageException usage; - usage << "The following option"; - usage << ((1 == nmissing) ? " is" : "s are"); - usage << " required:"; - for (size_t i = 0; i != missingLong.size(); ++i) - usage << " --" << missingLong[i]; - for (size_t i = 0; i != missingShort.size(); ++i) - usage << " -" << missingShort[i]; - throw usage; // don't need lineinfo, just the message - } - - void TOptsParser::PrintUsage(IOutputStream& os, const NColorizer::TColors& colors) const { - Opts_->PrintUsage(ProgramName(), os, colors); - } - - void TOptsParser::PrintUsage(IOutputStream& os) const { - PrintUsage(os, NColorizer::AutoColors(os)); - } + // also indicates subsequent options, if any, haven't been seen actually + OptsSeen_.clear(); + + const size_t nmissing = missingLong.size() + missingShort.size(); + if (0 == nmissing) + return; + + TUsageException usage; + usage << "The following option"; + usage << ((1 == nmissing) ? " is" : "s are"); + usage << " required:"; + for (size_t i = 0; i != missingLong.size(); ++i) + usage << " --" << missingLong[i]; + for (size_t i = 0; i != missingShort.size(); ++i) + usage << " -" << missingShort[i]; + throw usage; // don't need lineinfo, just the message + } + + void TOptsParser::PrintUsage(IOutputStream& os, const NColorizer::TColors& colors) const { + Opts_->PrintUsage(ProgramName(), os, colors); + } + + void TOptsParser::PrintUsage(IOutputStream& os) const { + PrintUsage(os, NColorizer::AutoColors(os)); + } } diff --git a/library/cpp/getopt/small/last_getopt_parser.h b/library/cpp/getopt/small/last_getopt_parser.h index 2cf8a6c308..6b5c8cbb97 100644 --- a/library/cpp/getopt/small/last_getopt_parser.h +++ b/library/cpp/getopt/small/last_getopt_parser.h @@ -8,85 +8,85 @@ #include <util/generic/list.h> namespace NLastGetopt { - /** + /** * NLastGetopt::TOptsParser is an implementation of parsing * argv/argv into TOptsParseResult by rules of TOpts. * * The class allows to make complicated handlers. * Note, that if PERMUTE mode is on, then data, pointed by argv can be changed. */ - class TOptsParser { - enum EIsOpt { - EIO_NONE, //is not an option name - EIO_SDASH, //single-dashed ('-c') option name - EIO_DDASH, //double-dashed ("--opt") option name - EIO_PLUS, //plus prefix ("+opt") option name - }; + class TOptsParser { + enum EIsOpt { + EIO_NONE, //is not an option name + EIO_SDASH, //single-dashed ('-c') option name + EIO_DDASH, //double-dashed ("--opt") option name + EIO_PLUS, //plus prefix ("+opt") option name + }; - public: // TODO: make private - const TOpts* Opts_; //rules of parsing + public: // TODO: make private + const TOpts* Opts_; //rules of parsing - // argc/argv pair - size_t Argc_; - const char** Argv_; + // argc/argv pair + size_t Argc_; + const char** Argv_; - private: - //the storage of last unkown options. TODO: can be moved to local-method scope - TCopyPtr<TOpt> TempCurrentOpt_; + private: + //the storage of last unkown options. TODO: can be moved to local-method scope + TCopyPtr<TOpt> TempCurrentOpt_; - public: - //storage of argv[0] - TString ProgramName_; + public: + //storage of argv[0] + TString ProgramName_; - //state of parsing: + //state of parsing: - size_t Pos_; // current element withing argv - size_t Sop_; // current char within arg - bool Stopped_; - bool GotMinusMinus_; //true if "--" have been seen in argv + size_t Pos_; // current element withing argv + size_t Sop_; // current char within arg + bool Stopped_; + bool GotMinusMinus_; //true if "--" have been seen in argv - protected: - const TOpt* CurrentOpt_; // ptr on the last meeted option - TStringBuf CurrentValue_; // the value of the last met argument (corresponding to CurrentOpt_) + protected: + const TOpt* CurrentOpt_; // ptr on the last meeted option + TStringBuf CurrentValue_; // the value of the last met argument (corresponding to CurrentOpt_) - private: - typedef THashSet<const TOpt*> TdOptSet; - TdOptSet OptsSeen_; //the set of options that have been met during parsing + private: + typedef THashSet<const TOpt*> TdOptSet; + TdOptSet OptsSeen_; //the set of options that have been met during parsing - TList<const TOpt*> OptsDefault_; + TList<const TOpt*> OptsDefault_; - private: - void Init(const TOpts* options, int argc, const char* argv[]); - void Init(const TOpts* options, int argc, char* argv[]); + private: + void Init(const TOpts* options, int argc, const char* argv[]); + void Init(const TOpts* options, int argc, char* argv[]); - bool CommitEndOfOptions(size_t pos); - bool Commit(const TOpt* currentOption, const TStringBuf& currentValue, size_t pos, size_t sop); + bool CommitEndOfOptions(size_t pos); + bool Commit(const TOpt* currentOption, const TStringBuf& currentValue, size_t pos, size_t sop); - bool ParseShortOptArg(size_t pos); - bool ParseOptArg(size_t pos); - bool ParseOptParam(const TOpt* opt, size_t pos); - bool ParseUnknownShortOptWithinArg(size_t pos, size_t sop); - bool ParseShortOptWithinArg(size_t pos, size_t sop); - bool ParseWithPermutation(); + bool ParseShortOptArg(size_t pos); + bool ParseOptArg(size_t pos); + bool ParseOptParam(const TOpt* opt, size_t pos); + bool ParseUnknownShortOptWithinArg(size_t pos, size_t sop); + bool ParseShortOptWithinArg(size_t pos, size_t sop); + bool ParseWithPermutation(); - bool DoNext(); - void Finish(); + bool DoNext(); + void Finish(); - EIsOpt IsOpt(const TStringBuf& arg) const; + EIsOpt IsOpt(const TStringBuf& arg) const; - void Swap(TOptsParser& that); + void Swap(TOptsParser& that); - public: - TOptsParser(const TOpts* options, int argc, const char* argv[]) { - Init(options, argc, argv); - } + public: + TOptsParser(const TOpts* options, int argc, const char* argv[]) { + Init(options, argc, argv); + } - TOptsParser(const TOpts* options, int argc, char* argv[]) { - Init(options, argc, argv); - } + TOptsParser(const TOpts* options, int argc, char* argv[]) { + Init(options, argc, argv); + } - /// fetch next argument, false if no more arguments left - bool Next(); + /// fetch next argument, false if no more arguments left + bool Next(); bool Seen(const TOpt* opt) const { return OptsSeen_.contains(opt); @@ -108,47 +108,47 @@ namespace NLastGetopt { } } - const TOpt* CurOpt() const { - return CurrentOpt_; - } - - const char* CurVal() const { - return CurrentValue_.data(); - } - - const TStringBuf& CurValStr() const { - return CurrentValue_; - } - - TStringBuf CurValOrOpt() const { - TStringBuf val(CurValStr()); - if (!val.IsInited() && CurOpt()->HasOptionalValue()) - val = CurOpt()->GetOptionalValue(); - return val; - } - - TStringBuf CurValOrDef(bool useDef = true) const { - TStringBuf val(CurValOrOpt()); - if (!val.IsInited() && useDef && CurOpt()->HasDefaultValue()) - val = CurOpt()->GetDefaultValue(); - return val; - } - - // true if this option was actually specified by the user - bool IsExplicit() const { - return nullptr == CurrentOpt_ || !OptsSeen_.empty(); - } - - bool CurrentIs(const TString& name) const { - return CurOpt()->NameIs(name); - } - - const TString& ProgramName() const { - return ProgramName_; - } - - void PrintUsage(IOutputStream& os = Cout) const; - - void PrintUsage(IOutputStream& os, const NColorizer::TColors& colors) const; - }; -} //namespace NLastGetopt + const TOpt* CurOpt() const { + return CurrentOpt_; + } + + const char* CurVal() const { + return CurrentValue_.data(); + } + + const TStringBuf& CurValStr() const { + return CurrentValue_; + } + + TStringBuf CurValOrOpt() const { + TStringBuf val(CurValStr()); + if (!val.IsInited() && CurOpt()->HasOptionalValue()) + val = CurOpt()->GetOptionalValue(); + return val; + } + + TStringBuf CurValOrDef(bool useDef = true) const { + TStringBuf val(CurValOrOpt()); + if (!val.IsInited() && useDef && CurOpt()->HasDefaultValue()) + val = CurOpt()->GetDefaultValue(); + return val; + } + + // true if this option was actually specified by the user + bool IsExplicit() const { + return nullptr == CurrentOpt_ || !OptsSeen_.empty(); + } + + bool CurrentIs(const TString& name) const { + return CurOpt()->NameIs(name); + } + + const TString& ProgramName() const { + return ProgramName_; + } + + void PrintUsage(IOutputStream& os = Cout) const; + + void PrintUsage(IOutputStream& os, const NColorizer::TColors& colors) const; + }; +} //namespace NLastGetopt diff --git a/library/cpp/getopt/small/last_getopt_support.h b/library/cpp/getopt/small/last_getopt_support.h index 17bed3e614..1f3a852382 100644 --- a/library/cpp/getopt/small/last_getopt_support.h +++ b/library/cpp/getopt/small/last_getopt_support.h @@ -1,178 +1,178 @@ #pragma once -#include <util/string/cast.h> +#include <util/string/cast.h> #include <util/generic/string.h> -#include <util/generic/vector.h> +#include <util/generic/vector.h> #include <util/generic/utility.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> namespace NLastGetopt { - class TOpt; - class TOpts; - class TOptsParser; - class TOptsParseResult; + class TOpt; + class TOpts; + class TOptsParser; + class TOptsParseResult; - /// base of all getopt exceptions - class TException: public yexception { - }; + /// base of all getopt exceptions + class TException: public yexception { + }; /// TOpts configuration is incorrect - class TConfException: public TException { - }; + class TConfException: public TException { + }; /// User passed incorrect arguments, parsing failed /// Note: use `throw TUsageException()` instead of `ythrow TUsageException()` to prevent appearence of stacktrace /// and location of the `ythrow` statment in error messages. - class TUsageException: public TException { - }; - - struct IOptHandler { - virtual void HandleOpt(const TOptsParser* parser) = 0; - virtual ~IOptHandler() = default; - }; - - namespace NPrivate { - template <typename TpFunc> - class THandlerFunctor0 - : public IOptHandler { - TpFunc Func_; - - public: - THandlerFunctor0(TpFunc func) - : Func_(func) - { - } - - void HandleOpt(const TOptsParser*) override { - Func_(); - } - }; - - template <typename TpFunc, typename TpArg = const TOptsParser*> - class THandlerFunctor1 - : public IOptHandler { - TpFunc Func_; - const TpArg Def_; - const bool HasDef_; - - public: - THandlerFunctor1(TpFunc func) - : Func_(func) - , Def_() - , HasDef_(false) - { - } - - template <typename T> - THandlerFunctor1(const TpFunc& func, const T& def) - : Func_(func) - , Def_(def) - , HasDef_(true) - { - } - - void HandleOpt(const TOptsParser* parser) override; - }; - - template <typename TpFunc> - class THandlerFunctor1<TpFunc, const TOptsParser*> - : public IOptHandler { - TpFunc Func_; - - public: - THandlerFunctor1(TpFunc func) - : Func_(func) - { - } - - void HandleOpt(const TOptsParser* parser) override { - Func_(parser); - } - }; - - template <typename T, typename TpVal = T> - class TStoreResultFunctor { - private: - T* Target_; - - public: - TStoreResultFunctor(T* target) - : Target_(target) - { - } - - void operator()(const TpVal& val) { - *Target_ = val; - } - }; - - template <typename TpTarget, typename TpFunc, typename TpVal = TpTarget> - class TStoreMappedResultFunctor { - private: - TpTarget* Target_; - const TpFunc Func_; - - public: - TStoreMappedResultFunctor(TpTarget* target, const TpFunc& func) - : Target_(target) - , Func_(func) - { - } - - void operator()(const TpVal& val) { - *Target_ = Func_(val); - } - }; - - template <typename T, typename TpVal = T> - class TStoreValueFunctor { - T* Target; - const TpVal Value; - - public: - template <typename TpArg> - TStoreValueFunctor(T* target, const TpArg& value) - : Target(target) - , Value(value) - { - } - - void operator()(const TOptsParser*) { - *Target = Value; - } - }; - - TString OptToString(char c); - TString OptToString(const TString& longOption); - TString OptToString(const TOpt* opt); - - template <typename T> - inline T OptFromStringImpl(const TStringBuf& value) { - return FromString<T>(value); + class TUsageException: public TException { + }; + + struct IOptHandler { + virtual void HandleOpt(const TOptsParser* parser) = 0; + virtual ~IOptHandler() = default; + }; + + namespace NPrivate { + template <typename TpFunc> + class THandlerFunctor0 + : public IOptHandler { + TpFunc Func_; + + public: + THandlerFunctor0(TpFunc func) + : Func_(func) + { + } + + void HandleOpt(const TOptsParser*) override { + Func_(); + } + }; + + template <typename TpFunc, typename TpArg = const TOptsParser*> + class THandlerFunctor1 + : public IOptHandler { + TpFunc Func_; + const TpArg Def_; + const bool HasDef_; + + public: + THandlerFunctor1(TpFunc func) + : Func_(func) + , Def_() + , HasDef_(false) + { + } + + template <typename T> + THandlerFunctor1(const TpFunc& func, const T& def) + : Func_(func) + , Def_(def) + , HasDef_(true) + { + } + + void HandleOpt(const TOptsParser* parser) override; + }; + + template <typename TpFunc> + class THandlerFunctor1<TpFunc, const TOptsParser*> + : public IOptHandler { + TpFunc Func_; + + public: + THandlerFunctor1(TpFunc func) + : Func_(func) + { + } + + void HandleOpt(const TOptsParser* parser) override { + Func_(parser); + } + }; + + template <typename T, typename TpVal = T> + class TStoreResultFunctor { + private: + T* Target_; + + public: + TStoreResultFunctor(T* target) + : Target_(target) + { + } + + void operator()(const TpVal& val) { + *Target_ = val; + } + }; + + template <typename TpTarget, typename TpFunc, typename TpVal = TpTarget> + class TStoreMappedResultFunctor { + private: + TpTarget* Target_; + const TpFunc Func_; + + public: + TStoreMappedResultFunctor(TpTarget* target, const TpFunc& func) + : Target_(target) + , Func_(func) + { + } + + void operator()(const TpVal& val) { + *Target_ = Func_(val); + } + }; + + template <typename T, typename TpVal = T> + class TStoreValueFunctor { + T* Target; + const TpVal Value; + + public: + template <typename TpArg> + TStoreValueFunctor(T* target, const TpArg& value) + : Target(target) + , Value(value) + { + } + + void operator()(const TOptsParser*) { + *Target = Value; + } + }; + + TString OptToString(char c); + TString OptToString(const TString& longOption); + TString OptToString(const TOpt* opt); + + template <typename T> + inline T OptFromStringImpl(const TStringBuf& value) { + return FromString<T>(value); } - template <> - inline TStringBuf OptFromStringImpl<TStringBuf>(const TStringBuf& value) { - return value; - } - - template <> - inline const char* OptFromStringImpl<const char*>(const TStringBuf& value) { - return value.data(); - } - - template <typename T, typename TSomeOpt> - T OptFromString(const TStringBuf& value, const TSomeOpt opt) { - try { - return OptFromStringImpl<T>(value); - } catch (...) { + template <> + inline TStringBuf OptFromStringImpl<TStringBuf>(const TStringBuf& value) { + return value; + } + + template <> + inline const char* OptFromStringImpl<const char*>(const TStringBuf& value) { + return value.data(); + } + + template <typename T, typename TSomeOpt> + T OptFromString(const TStringBuf& value, const TSomeOpt opt) { + try { + return OptFromStringImpl<T>(value); + } catch (...) { throw TUsageException() << "failed to parse opt " << OptToString(opt) << " value " << TString(value).Quote() << ": " << CurrentExceptionMessage(); - } - } - - // wrapper of FromString<T> that prints nice message about option used - template <typename T, typename TSomeOpt> - T OptFromString(const TStringBuf& value, const TSomeOpt opt); - - } -} + } + } + + // wrapper of FromString<T> that prints nice message about option used + template <typename T, typename TSomeOpt> + T OptFromString(const TStringBuf& value, const TSomeOpt opt); + + } +} diff --git a/library/cpp/getopt/small/modchooser.cpp b/library/cpp/getopt/small/modchooser.cpp index 2fa5cfd070..4ae8c90dae 100644 --- a/library/cpp/getopt/small/modchooser.cpp +++ b/library/cpp/getopt/small/modchooser.cpp @@ -6,7 +6,7 @@ #include <library/cpp/colorizer/colors.h> -#include <util/stream/output.h> +#include <util/stream/output.h> #include <util/stream/format.h> #include <util/generic/yexception.h> #include <util/generic/ptr.h> diff --git a/library/cpp/getopt/small/modchooser.h b/library/cpp/getopt/small/modchooser.h index 0a8de6d50b..8717acdd54 100644 --- a/library/cpp/getopt/small/modchooser.h +++ b/library/cpp/getopt/small/modchooser.h @@ -2,7 +2,7 @@ #include "last_getopt_opts.h" -#include <util/generic/map.h> +#include <util/generic/map.h> #include <util/generic/string.h> #include <util/generic/vector.h> @@ -19,7 +19,7 @@ using TMainFunctionRawPtr = int (*)(const int argc, const char** argv); //! Mode class with vector of cli arguments. class TMainClassV { public: - virtual int operator()(const TVector<TString>& argv) = 0; + virtual int operator()(const TVector<TString>& argv) = 0; virtual ~TMainClassV() = default; }; @@ -45,7 +45,7 @@ class TModChooser { public: TModChooser(); ~TModChooser(); - + public: void AddMode(const TString& mode, TMainFunctionRawPtr func, const TString& description, bool hidden = false, bool noCompletion = false); void AddMode(const TString& mode, TMainFunctionRawPtrV func, const TString& description, bool hidden = false, bool noCompletion = false); @@ -98,7 +98,7 @@ public: int Run(int argc, const char** argv) const; //! Run appropriate mode. Same as Run(const int, const char**) - int Run(const TVector<TString>& argv) const; + int Run(const TVector<TString>& argv) const; void PrintHelp(const TString& progName) const; diff --git a/library/cpp/getopt/small/opt.cpp b/library/cpp/getopt/small/opt.cpp index 744501765c..36b083ce2a 100644 --- a/library/cpp/getopt/small/opt.cpp +++ b/library/cpp/getopt/small/opt.cpp @@ -1,13 +1,13 @@ -#include "opt.h" - -#include <util/system/progname.h> +#include "opt.h" +#include <util/system/progname.h> + #include <ctype.h> using namespace NLastGetopt; namespace { - struct TOptsNoDefault: public TOpts { + struct TOptsNoDefault: public TOpts { TOptsNoDefault(const TStringBuf& optstring = TStringBuf()) : TOpts(optstring) { @@ -48,7 +48,7 @@ Opt::Opt(int argc, char* argv[], const char* optString, const Ion* longOptions, } Opt::Opt(int argc, const char* argv[], const char* optString, const Ion* longOptions, bool longOnly, bool isOpen) { - Init(argc, (char**)argv, optString, longOptions, longOnly, isOpen); + Init(argc, (char**)argv, optString, longOptions, longOnly, isOpen); } int Opt::Get() { @@ -58,7 +58,7 @@ int Opt::Get() { int Opt::Get(int* longOptionIndex) { if (GotError_) return EOF; - + Arg = nullptr; try { @@ -67,12 +67,12 @@ int Opt::Get(int* longOptionIndex) { if (!r) { return EOF; } else { - Arg = (char*)OptsParser_->CurVal(); + Arg = (char*)OptsParser_->CurVal(); if (!OptsParser_->CurOpt()) { // possible if RETURN_IN_ORDER return 1; } else { - const Ion* ion = (const Ion*)OptsParser_->CurOpt()->UserValue(); + const Ion* ion = (const Ion*)OptsParser_->CurOpt()->UserValue(); if (longOptionIndex) { *longOptionIndex = int(ion - Ions_); } @@ -100,20 +100,20 @@ const char** Opt::GetArgV() const { return OptsParser_->Argv_; } -int opt_get_number(int& argc, char* argv[]) { - int num = -1; - for (int a = 1; a < argc; a++) { - if (*argv[a] == '-' && isdigit((ui8)argv[a][1])) { - char* ne; - num = strtol(argv[a] + 1, &ne, 10); - if (*ne) { - memmove(argv[a] + 1, ne, strlen(ne) + 1); - } else { - for (argc--; a < argc; a++) - argv[a] = argv[a + 1]; - } - break; - } - } - return num; +int opt_get_number(int& argc, char* argv[]) { + int num = -1; + for (int a = 1; a < argc; a++) { + if (*argv[a] == '-' && isdigit((ui8)argv[a][1])) { + char* ne; + num = strtol(argv[a] + 1, &ne, 10); + if (*ne) { + memmove(argv[a] + 1, ne, strlen(ne) + 1); + } else { + for (argc--; a < argc; a++) + argv[a] = argv[a + 1]; + } + break; + } + } + return num; } diff --git a/library/cpp/getopt/small/opt.h b/library/cpp/getopt/small/opt.h index ecb57439bc..1bbd3811a3 100644 --- a/library/cpp/getopt/small/opt.h +++ b/library/cpp/getopt/small/opt.h @@ -1,8 +1,8 @@ #pragma once -#include "last_getopt.h" - -#include <util/generic/ptr.h> +#include "last_getopt.h" + +#include <util/generic/ptr.h> #include <util/generic/noncopyable.h> // implementation of Opt class using last getopt @@ -54,20 +54,20 @@ */ #define OPT_RETURN_IN_ORDER "-" -#define OPT_REQUIRE_ORDER "+" -#define OPT_DONT_STORE_ARG ((void*)0) +#define OPT_REQUIRE_ORDER "+" +#define OPT_DONT_STORE_ARG ((void*)0) -class Opt : TNonCopyable { +class Opt : TNonCopyable { public: - enum HasArg { WithoutArg, - WithArg, - PossibleArg }; + enum HasArg { WithoutArg, + WithArg, + PossibleArg }; struct Ion { - const char* name; - HasArg has_arg; - int* flag; - int val; + const char* name; + HasArg has_arg; + int* flag; + int val; }; private: @@ -75,7 +75,7 @@ private: THolder<NLastGetopt::TOptsParser> OptsParser_; const Ion* Ions_; bool GotError_; - + void Init(int argc, char* argv[], const char* optString, const Ion* longOptions = nullptr, bool longOnly = false, bool isOpen = false); public: @@ -89,19 +89,19 @@ public: return Get(); } - const char* GetArg() const { - return Arg; - } + const char* GetArg() const { + return Arg; + } TVector<TString> GetFreeArgs() const { return NLastGetopt::TOptsParseResult(&*Opts_, GetArgC(), GetArgV()).GetFreeArgs(); } // obsolete, use GetArg() instead - char* Arg; /* option argument if any or NULL */ + char* Arg; /* option argument if any or NULL */ - int Ind; /* command line index */ - bool Err; /* flag to print error messages */ + int Ind; /* command line index */ + bool Err; /* flag to print error messages */ int GetArgC() const; const char** GetArgV() const; @@ -111,32 +111,32 @@ public: // call before getopt. returns non-negative int, removing it from arguments (not found: -1) // Example: returns 11 for "progname -11abc", -1 for "progname -a11" -int opt_get_number(int& argc, char* argv[]); - -#define OPTION_HANDLING_PROLOG \ - { \ - int optlet; \ - while (EOF != (optlet = opt.Get())) { \ - switch (optlet) { -#define OPTION_HANDLING_PROLOG_ANON(S) \ - { \ - Opt opt(argc, argv, (S)); \ - int optlet; \ - while (EOF != (optlet = opt.Get())) { \ - switch (optlet) { -#define OPTION_HANDLE_BEGIN(opt) case opt: { -#define OPTION_HANDLE_END \ - } \ - break; +int opt_get_number(int& argc, char* argv[]); + +#define OPTION_HANDLING_PROLOG \ + { \ + int optlet; \ + while (EOF != (optlet = opt.Get())) { \ + switch (optlet) { +#define OPTION_HANDLING_PROLOG_ANON(S) \ + { \ + Opt opt(argc, argv, (S)); \ + int optlet; \ + while (EOF != (optlet = opt.Get())) { \ + switch (optlet) { +#define OPTION_HANDLE_BEGIN(opt) case opt: { +#define OPTION_HANDLE_END \ + } \ + break; #define OPTION_HANDLE(opt, handle) \ - OPTION_HANDLE_BEGIN(opt) \ - handle; \ - OPTION_HANDLE_END - -#define OPTION_HANDLING_EPILOG \ - default: \ - ythrow yexception() << "unknown optlet"; \ - } \ - } \ - } + OPTION_HANDLE_BEGIN(opt) \ + handle; \ + OPTION_HANDLE_END + +#define OPTION_HANDLING_EPILOG \ + default: \ + ythrow yexception() << "unknown optlet"; \ + } \ + } \ + } diff --git a/library/cpp/getopt/small/opt2.cpp b/library/cpp/getopt/small/opt2.cpp index 0cdc774e78..66a17e2e5d 100644 --- a/library/cpp/getopt/small/opt2.cpp +++ b/library/cpp/getopt/small/opt2.cpp @@ -1,21 +1,21 @@ -#include "opt2.h" - -#include <util/generic/hash.h> -#include <util/generic/utility.h> +#include "opt2.h" + +#include <util/generic/hash.h> +#include <util/generic/utility.h> #include <util/generic/yexception.h> #include <util/str_stl.h> -#include <stdio.h> -#include <errno.h> -#include <ctype.h> - +#include <stdio.h> +#include <errno.h> +#include <ctype.h> + void Opt2::Clear() { Specs.clear(); memset(SpecsMap, 0, sizeof(SpecsMap)); Pos.clear(); } -void Opt2::Init(int argc, char* const* argv, const char* optspec, IntRange free_args_num, const char* long_alias) { +void Opt2::Init(int argc, char* const* argv, const char* optspec, IntRange free_args_num, const char* long_alias) { Clear(); Argc = argc; Argv = argv; @@ -29,7 +29,7 @@ void Opt2::Init(int argc, char* const* argv, const char* optspec, IntRange free_ BadPosCount = HasErrors = true; } -void Opt2::EatArgv(const char* optspec, const char* long_alias) { +void Opt2::EatArgv(const char* optspec, const char* long_alias) { // some flags bool require_order = false; if (*optspec == '+') { @@ -39,11 +39,11 @@ void Opt2::EatArgv(const char* optspec, const char* long_alias) { if (*optspec == '-') ythrow yexception() << "Flag '-' can not be used in Opt2's optspec"; // step 1 - parse optspec - for (const char* s = optspec; *s; s++) { + for (const char* s = optspec; *s; s++) { if (SpecsMap[(ui8)*s]) - ythrow yexception() << "Symbol '" << *s << "' is met twice in Opt2's optspec"; + ythrow yexception() << "Symbol '" << *s << "' is met twice in Opt2's optspec"; if (*s == '?' || *s == '-') - ythrow yexception() << "Opt2: Symbol '" << *s << "' can not be used in optspec because it is reserved"; + ythrow yexception() << "Opt2: Symbol '" << *s << "' can not be used in optspec because it is reserved"; Specs.push_back(Opt2Param()); SpecsMap[(ui8)*s] = (ui8)Specs.size(); // actual index + 1 Specs.back().opt = *s; @@ -59,27 +59,27 @@ void Opt2::EatArgv(const char* optspec, const char* long_alias) { // with single short option (extend it if you really need). THashMap<const char*, char> long2short; long2short["help"] = '?'; - long_alias = long_alias ? long_alias : ""; + long_alias = long_alias ? long_alias : ""; alias_copy = long_alias; - for (char* s = alias_copy.begin(); s && *s;) { - char* eq = strchr(s, '='); - char* comma = strchr(s, ','); - if (comma) - *comma = 0; + for (char* s = alias_copy.begin(); s && *s;) { + char* eq = strchr(s, '='); + char* comma = strchr(s, ','); + if (comma) + *comma = 0; if (!eq || (comma && comma < eq)) - ythrow yexception() << "Opt2, long_alias: '=' is expected after " << s; + ythrow yexception() << "Opt2, long_alias: '=' is expected after " << s; *eq++ = 0; if (!*eq || eq[1]) - ythrow yexception() << "Opt2, long_alias: single letter must be assigned to " << s; + ythrow yexception() << "Opt2, long_alias: single letter must be assigned to " << s; if (!SpecsMap[(ui8)*eq]) - ythrow yexception() << "Opt2, long_alias: trying to assign unknown option '" << *eq << "' to " << s; - Opt2Param& p = Specs[SpecsMap[(ui8)*eq] - 1]; + ythrow yexception() << "Opt2, long_alias: trying to assign unknown option '" << *eq << "' to " << s; + Opt2Param& p = Specs[SpecsMap[(ui8)*eq] - 1]; // If several long options aliased to some letter, only last one is shown in usage p.LongOptName = s; if (long2short.find(s) != long2short.end()) - ythrow yexception() << "Opt2, long_alias: " << s << " specified twice"; + ythrow yexception() << "Opt2, long_alias: " << s << " specified twice"; long2short[s] = *eq; - s = comma ? comma + 1 : nullptr; + s = comma ? comma + 1 : nullptr; } if (Argc < 1) { @@ -96,7 +96,7 @@ void Opt2::EatArgv(const char* optspec, const char* long_alias) { Pos.push_back(Argv[ind]); continue; } - const char* s = Argv[ind] + 1; + const char* s = Argv[ind] + 1; if (*s == '-') { if (!*++s) { // `--' terminates the list of options @@ -104,7 +104,7 @@ void Opt2::EatArgv(const char* optspec, const char* long_alias) { break; } // long option always spans one argv (--switch or --option-name=value) - const char* eq = strchr(s, '='); + const char* eq = strchr(s, '='); TString lname(s, eq ? (size_t)(eq - s) : (size_t)strlen(s)); THashMap<const char*, char>::iterator i = long2short.find(lname.data()); if (i == long2short.end()) { @@ -117,7 +117,7 @@ void Opt2::EatArgv(const char* optspec, const char* long_alias) { HasErrors = true; continue; } - Opt2Param& p = Specs[SpecsMap[(ui8)i->second] - 1]; + Opt2Param& p = Specs[SpecsMap[(ui8)i->second] - 1]; p.IsFound = true; if (p.HasArg && !eq) { HasErrors = true; @@ -142,11 +142,11 @@ void Opt2::EatArgv(const char* optspec, const char* long_alias) { continue; return; } - Opt2Param& p = Specs[SpecsMap[(ui8)*s] - 1]; + Opt2Param& p = Specs[SpecsMap[(ui8)*s] - 1]; p.IsFound = true; if (p.HasArg) { - if (s[1]) - p.ActualValue.push_back(s + 1); + if (s[1]) + p.ActualValue.push_back(s + 1); else { ind++; if (ind == Argc) { @@ -165,10 +165,10 @@ void Opt2::EatArgv(const char* optspec, const char* long_alias) { Pos.push_back(Argv[ind]); } -Opt2Param& Opt2::GetInternal(char opt, const char* defValue, const char* helpUsage, bool requred) { +Opt2Param& Opt2::GetInternal(char opt, const char* defValue, const char* helpUsage, bool requred) { if (!SpecsMap[(ui8)opt]) - ythrow yexception() << "Unspecified option character '" << opt << "' asked from Opt2::Get"; - Opt2Param& p = Specs[SpecsMap[(ui8)opt] - 1]; + ythrow yexception() << "Unspecified option character '" << opt << "' asked from Opt2::Get"; + Opt2Param& p = Specs[SpecsMap[(ui8)opt] - 1]; p.DefValue = defValue; p.HelpUsage = helpUsage; p.IsRequired = requred; @@ -180,49 +180,49 @@ Opt2Param& Opt2::GetInternal(char opt, const char* defValue, const char* helpUsa } // For options with parameters -const char* Opt2::Arg(char opt, const char* help, const char* def, bool required) { - Opt2Param& p = GetInternal(opt, def, help, required); +const char* Opt2::Arg(char opt, const char* help, const char* def, bool required) { + Opt2Param& p = GetInternal(opt, def, help, required); if (!p.HasArg) - ythrow yexception() << "Opt2::Arg called for '" << opt << "' which is an option without argument"; - return p.IsFound ? p.ActualValue.empty() ? nullptr : p.ActualValue.back() : def; + ythrow yexception() << "Opt2::Arg called for '" << opt << "' which is an option without argument"; + return p.IsFound ? p.ActualValue.empty() ? nullptr : p.ActualValue.back() : def; } // For options with parameters -const char* Opt2::Arg(char opt, const char* help, TString def, bool required) { - Opt2Param& p = GetInternal(opt, nullptr, help, required); +const char* Opt2::Arg(char opt, const char* help, TString def, bool required) { + Opt2Param& p = GetInternal(opt, nullptr, help, required); if (!p.HasArg) - ythrow yexception() << "Opt2::Arg called for '" << opt << "' which is an option without argument"; + ythrow yexception() << "Opt2::Arg called for '" << opt << "' which is an option without argument"; p.DefValueStr = def; p.DefValue = p.DefValueStr.begin(); - return p.IsFound ? p.ActualValue.empty() ? nullptr : p.ActualValue.back() : p.DefValue; + return p.IsFound ? p.ActualValue.empty() ? nullptr : p.ActualValue.back() : p.DefValue; } // Options with parameters that can be specified several times -const TVector<const char*>& Opt2::MArg(char opt, const char* help) { - Opt2Param& p = GetInternal(opt, nullptr, help, false); +const TVector<const char*>& Opt2::MArg(char opt, const char* help) { + Opt2Param& p = GetInternal(opt, nullptr, help, false); p.MultipleUse = true; if (!p.HasArg) - ythrow yexception() << "Opt2::Arg called for '" << opt << "' which is an option without argument"; + ythrow yexception() << "Opt2::Arg called for '" << opt << "' which is an option without argument"; return p.ActualValue; } /// For options w/o parameters -bool Opt2::Has(char opt, const char* help) { - Opt2Param& p = GetInternal(opt, nullptr, help, false); +bool Opt2::Has(char opt, const char* help) { + Opt2Param& p = GetInternal(opt, nullptr, help, false); if (p.HasArg) - ythrow yexception() << "Opt2::Has called for '" << opt << "' which is an option with argument"; + ythrow yexception() << "Opt2::Has called for '" << opt << "' which is an option with argument"; return p.IsFound; } // Get() + strtol, may set up HasErrors -long Opt2::Int(char opt, const char* help, long def, bool required) { - Opt2Param& p = GetInternal(opt, (char*)(uintptr_t)def, help, required); +long Opt2::Int(char opt, const char* help, long def, bool required) { + Opt2Param& p = GetInternal(opt, (char*)(uintptr_t)def, help, required); if (!p.HasArg) - ythrow yexception() << "Opt2::Int called for '" << opt << "' which is an option without argument"; + ythrow yexception() << "Opt2::Int called for '" << opt << "' which is an option without argument"; p.IsNumeric = true; if (!p.IsFound || p.ActualValue.empty() || !p.ActualValue.back()) return def; - char* e; + char* e; long rv = strtol(p.ActualValue.back(), &e, 10); if (e == p.ActualValue.back() || *e) { OptionWrongArg = opt; @@ -232,14 +232,14 @@ long Opt2::Int(char opt, const char* help, long def, bool required) { } // Get() + strtoul, may set up HasErrors -unsigned long Opt2::UInt(char opt, const char* help, unsigned long def, bool required) { - Opt2Param& p = GetInternal(opt, (char*)(uintptr_t)def, help, required); +unsigned long Opt2::UInt(char opt, const char* help, unsigned long def, bool required) { + Opt2Param& p = GetInternal(opt, (char*)(uintptr_t)def, help, required); if (!p.HasArg) - ythrow yexception() << "Opt2::UInt called for '" << opt << "' which is an option without argument"; + ythrow yexception() << "Opt2::UInt called for '" << opt << "' which is an option without argument"; p.IsNumeric = true; if (!p.IsFound || p.ActualValue.empty() || !p.ActualValue.back()) return def; - char* e; + char* e; unsigned long rv = strtoul(p.ActualValue.back(), &e, 10); if (e == p.ActualValue.back() || *e) { OptionWrongArg = opt; @@ -251,14 +251,14 @@ unsigned long Opt2::UInt(char opt, const char* help, unsigned long def, bool req // Add user defined error message and set error flag void Opt2::AddError(const char* message) { HasErrors = true; - if (message) + if (message) UserErrorMessages.push_back(message); } -int Opt2::AutoUsage(const char* free_arg_names) { +int Opt2::AutoUsage(const char* free_arg_names) { if (!HasErrors) return 0; - FILE* where = UnknownOption == '?' ? stdout : stderr; + FILE* where = UnknownOption == '?' ? stdout : stderr; char req_str[256], nreq_str[256]; int req = 0, nreq = 0; for (int n = 0; n < (int)Specs.size(); n++) @@ -267,11 +267,11 @@ int Opt2::AutoUsage(const char* free_arg_names) { else nreq_str[nreq++] = Specs[n].opt; req_str[req] = 0, nreq_str[nreq] = 0; - const char* prog = strrchr(Argv[0], LOCSLASH_C); - prog = prog ? prog + 1 : Argv[0]; - fprintf(where, "Usage: %s%s%s%s%s%s%s%s\n", prog, req ? " -" : "", req_str, - nreq ? " [-" : "", nreq_str, nreq ? "]" : "", - free_arg_names && *free_arg_names ? " " : "", free_arg_names); + const char* prog = strrchr(Argv[0], LOCSLASH_C); + prog = prog ? prog + 1 : Argv[0]; + fprintf(where, "Usage: %s%s%s%s%s%s%s%s\n", prog, req ? " -" : "", req_str, + nreq ? " [-" : "", nreq_str, nreq ? "]" : "", + free_arg_names && *free_arg_names ? " " : "", free_arg_names); for (auto& spec : Specs) { const char* hlp = !spec.HelpUsage.empty() ? spec.HelpUsage.data() : spec.HasArg ? "<arg>" : ""; if (!spec.HasArg || spec.IsRequired) @@ -305,7 +305,7 @@ int Opt2::AutoUsage(const char* free_arg_names) { return UnknownOption == '?' ? 1 : 2; } -void Opt2::AutoUsageErr(const char* free_arg_names) { +void Opt2::AutoUsageErr(const char* free_arg_names) { if (AutoUsage(free_arg_names)) exit(1); } @@ -314,15 +314,15 @@ void Opt2::AutoUsageErr(const char* free_arg_names) { // TODO: convert it to unittest bool opt2_ut_fail = false, opt_ut_verbose = false; -const char* ut_optspec; -int ut_real(TString args, bool err_exp, const char* A_exp, int b_exp, bool a_exp, const char* p1_exp, const char* p2_exp) { - char* argv[32]; +const char* ut_optspec; +int ut_real(TString args, bool err_exp, const char* A_exp, int b_exp, bool a_exp, const char* p1_exp, const char* p2_exp) { + char* argv[32]; int argc = sf(' ', argv, args.begin()); Opt2 opt(argc, argv, ut_optspec, 2, "option-1=A,option-2=a,"); - const char* A = opt.Arg('A', "<qqq> - blah"); - int b = opt.Int('b', "<rrr> - blah", 2); - bool a = opt.Has('a', "- blah"); - /*const char *C = */ opt.Arg('C', "<ccc> - blah", 0); + const char* A = opt.Arg('A', "<qqq> - blah"); + int b = opt.Int('b', "<rrr> - blah", 2); + bool a = opt.Has('a', "- blah"); + /*const char *C = */ opt.Arg('C', "<ccc> - blah", 0); if (opt_ut_verbose) opt.AutoUsage(""); @@ -343,7 +343,7 @@ int ut_real(TString args, bool err_exp, const char* A_exp, int b_exp, bool a_exp return false; } -void ut(const char* args, bool err_exp, const char* A_exp, int b_exp, bool a_exp, const char* p1_exp, const char* p2_exp) { +void ut(const char* args, bool err_exp, const char* A_exp, int b_exp, bool a_exp, const char* p1_exp, const char* p2_exp) { if (opt_ut_verbose) fprintf(stderr, "Testing: %s\n", args); if (int rv = ut_real(args, err_exp, A_exp, b_exp, a_exp, p1_exp, p2_exp)) { @@ -355,7 +355,7 @@ void ut(const char* args, bool err_exp, const char* A_exp, int b_exp, bool a_exp } } -int main(int argc, char* argv[]) { +int main(int argc, char* argv[]) { Opt2 opt(argc, argv, "v", 0); opt_ut_verbose = opt.Has('v', "- some verboseness"); opt.AutoUsageErr(""); diff --git a/library/cpp/getopt/small/opt2.h b/library/cpp/getopt/small/opt2.h index 4d9d943237..9a9eb579a8 100644 --- a/library/cpp/getopt/small/opt2.h +++ b/library/cpp/getopt/small/opt2.h @@ -30,11 +30,11 @@ struct Opt2Param { bool IsNumeric; bool IsRequired; bool MultipleUse; - const char* DefValue; + const char* DefValue; TString DefValueStr; TString HelpUsage; TVector<const char*> ActualValue; - const char* LongOptName; + const char* LongOptName; Opt2Param() : HasArg(false) , IsFound(0) @@ -67,45 +67,45 @@ class Opt2 { public: Opt2() = default; - Opt2(int argc, char* const* argv, const char* optspec, IntRange free_args_num = -1, const char* long_alias = nullptr) { + Opt2(int argc, char* const* argv, const char* optspec, IntRange free_args_num = -1, const char* long_alias = nullptr) { Init(argc, argv, optspec, free_args_num, long_alias); } // Init throws exception only in case of incorrect optspec. // In other cases, consult HasErrors or call AutoUsage() - void Init(int argc, char* const* argv, const char* optspec, IntRange free_args_num = -1, const char* long_alias = nullptr); + void Init(int argc, char* const* argv, const char* optspec, IntRange free_args_num = -1, const char* long_alias = nullptr); // In case of incorrect options, constructs and prints Usage text, // usually to stderr (however, to stdout if '-?' switch was used), and returns 1. - int AutoUsage(const char* free_arg_names = ""); + int AutoUsage(const char* free_arg_names = ""); // same as AutoUsage but calls exit(1) instead of error code - void AutoUsageErr(const char* free_arg_names = ""); + void AutoUsageErr(const char* free_arg_names = ""); // For options with parameters - const char* Arg(char opt, const char* helpUsage, const char* defValue, bool required = false); - const char* Arg(char opt, const char* helpUsage) { + const char* Arg(char opt, const char* helpUsage, const char* defValue, bool required = false); + const char* Arg(char opt, const char* helpUsage) { return Arg(opt, helpUsage, nullptr, true); } - const char* Arg(char opt, const char* helpUsage, TString defValue, bool required = false); + const char* Arg(char opt, const char* helpUsage, TString defValue, bool required = false); // Options with parameters that can be specified several times - const TVector<const char*>& MArg(char opt, const char* helpUsage); + const TVector<const char*>& MArg(char opt, const char* helpUsage); // Get() + strtol, may set up HasErrors - long Int(char opt, const char* helpUsage, long defValue, bool required = false); - long Int(char opt, const char* helpUsage) { + long Int(char opt, const char* helpUsage, long defValue, bool required = false); + long Int(char opt, const char* helpUsage) { return Int(opt, helpUsage, 0, true); } // Get() + strtoul, may set up HasErrors - unsigned long UInt(char opt, const char* helpUsage, unsigned long defValue, bool required = false); - unsigned long UInt(char opt, const char* helpUsage) { + unsigned long UInt(char opt, const char* helpUsage, unsigned long defValue, bool required = false); + unsigned long UInt(char opt, const char* helpUsage) { return UInt(opt, helpUsage, 0, true); } // For options w/o parameters - bool Has(char opt, const char* helpUsage); + bool Has(char opt, const char* helpUsage); // Add user defined error message and set error flag void AddError(const char* message = nullptr); @@ -118,7 +118,7 @@ public: private: bool BadPosCount; char UnknownOption; - char* UnknownLongOption; + char* UnknownLongOption; char OptionMissingArg; char OptionWrongArg; char RequiredOptionMissing; @@ -131,7 +131,7 @@ protected: ui8 SpecsMap[256]; TVector<Opt2Param> Specs; TString alias_copy; - void EatArgv(const char* optspec, const char* long_alias); + void EatArgv(const char* optspec, const char* long_alias); void Clear(); - Opt2Param& GetInternal(char opt, const char* defValue, const char* helpUsage, bool required); + Opt2Param& GetInternal(char opt, const char* defValue, const char* helpUsage, bool required); }; diff --git a/library/cpp/getopt/small/posix_getopt.cpp b/library/cpp/getopt/small/posix_getopt.cpp index bd06f3499f..57cc5c66a7 100644 --- a/library/cpp/getopt/small/posix_getopt.cpp +++ b/library/cpp/getopt/small/posix_getopt.cpp @@ -1,7 +1,7 @@ -#include "posix_getopt.h" - -#include <util/generic/ptr.h> - +#include "posix_getopt.h" + +#include <util/generic/ptr.h> + #include <ctype.h> namespace NLastGetopt { @@ -14,8 +14,8 @@ namespace NLastGetopt { static THolder<TOpts> Opts; static THolder<TOptsParser> OptsParser; - int getopt_long_impl(int argc, char* const* argv, const char* optstring, - const struct option* longopts, int* longindex, bool long_only) { + int getopt_long_impl(int argc, char* const* argv, const char* optstring, + const struct option* longopts, int* longindex, bool long_only) { if (!Opts || optreset == 1) { optarg = nullptr; optind = 1; @@ -38,7 +38,7 @@ namespace NLastGetopt { opt->UserValue(o->flag); } - OptsParser.Reset(new TOptsParser(&*Opts, argc, (const char**)argv)); + OptsParser.Reset(new TOptsParser(&*Opts, argc, (const char**)argv)); } optarg = nullptr; @@ -47,10 +47,10 @@ namespace NLastGetopt { if (!OptsParser->Next()) { return -1; } else { - optarg = (char*)OptsParser->CurVal(); + optarg = (char*)OptsParser->CurVal(); optind = (int)OptsParser->Pos_; if (longindex && OptsParser->CurOpt()) - *longindex = (int)Opts->IndexOf(OptsParser->CurOpt()); + *longindex = (int)Opts->IndexOf(OptsParser->CurOpt()); return OptsParser->CurOpt() ? OptsParser->CurOpt()->GetCharOr0() : 1; } } catch (const NLastGetopt::TException&) { @@ -58,13 +58,13 @@ namespace NLastGetopt { } } - int getopt_long(int argc, char* const* argv, const char* optstring, - const struct option* longopts, int* longindex) { + int getopt_long(int argc, char* const* argv, const char* optstring, + const struct option* longopts, int* longindex) { return getopt_long_impl(argc, argv, optstring, longopts, longindex, false); } int getopt_long_only(int argc, char* const* argv, const char* optstring, - const struct option* longopts, int* longindex) { + const struct option* longopts, int* longindex) { return getopt_long_impl(argc, argv, optstring, longopts, longindex, true); } diff --git a/library/cpp/getopt/small/posix_getopt.h b/library/cpp/getopt/small/posix_getopt.h index e6af1e0284..8a8891a1f6 100644 --- a/library/cpp/getopt/small/posix_getopt.h +++ b/library/cpp/getopt/small/posix_getopt.h @@ -25,8 +25,8 @@ namespace NLastGetopt { }; int getopt(int argc, char* const* argv, const char* optstring); - int getopt_long(int argc, char* const* argv, const char* optstring, - const struct option* longopts, int* longindex); + int getopt_long(int argc, char* const* argv, const char* optstring, + const struct option* longopts, int* longindex); int getopt_long_only(int argc, char* const* argv, const char* optstring, - const struct option* longopts, int* longindex); + const struct option* longopts, int* longindex); } diff --git a/library/cpp/getopt/small/ygetopt.cpp b/library/cpp/getopt/small/ygetopt.cpp index 1f52827f74..8952faeeae 100644 --- a/library/cpp/getopt/small/ygetopt.cpp +++ b/library/cpp/getopt/small/ygetopt.cpp @@ -1,108 +1,108 @@ -#include "opt.h" -#include "ygetopt.h" - +#include "opt.h" +#include "ygetopt.h" + #include <util/generic/string.h> #include <util/generic/vector.h> #include <util/generic/yexception.h> - -class TGetOpt::TImpl: public TSimpleRefCount<TImpl> { -public: - inline TImpl(int argc, const char* const* argv, const TString& fmt) - : args(argv, argv + argc) - , format(fmt) - { - if (argc == 0) { - ythrow yexception() << "zero argc"; - } - } - - inline ~TImpl() = default; - - TVector<TString> args; - const TString format; -}; - -class TGetOpt::TIterator::TIterImpl: public TSimpleRefCount<TIterImpl> { -public: - inline TIterImpl(const TGetOpt* parent) - : Args_(parent->Impl_->args) - , ArgsPtrs_(new char*[Args_.size() + 1]) - , Format_(parent->Impl_->format) - , OptLet_(0) - , Arg_(nullptr) - { - for (size_t i = 0; i < Args_.size(); ++i) { - ArgsPtrs_.Get()[i] = Args_[i].begin(); - } - - ArgsPtrs_.Get()[Args_.size()] = nullptr; + +class TGetOpt::TImpl: public TSimpleRefCount<TImpl> { +public: + inline TImpl(int argc, const char* const* argv, const TString& fmt) + : args(argv, argv + argc) + , format(fmt) + { + if (argc == 0) { + ythrow yexception() << "zero argc"; + } + } + + inline ~TImpl() = default; + + TVector<TString> args; + const TString format; +}; + +class TGetOpt::TIterator::TIterImpl: public TSimpleRefCount<TIterImpl> { +public: + inline TIterImpl(const TGetOpt* parent) + : Args_(parent->Impl_->args) + , ArgsPtrs_(new char*[Args_.size() + 1]) + , Format_(parent->Impl_->format) + , OptLet_(0) + , Arg_(nullptr) + { + for (size_t i = 0; i < Args_.size(); ++i) { + ArgsPtrs_.Get()[i] = Args_[i].begin(); + } + + ArgsPtrs_.Get()[Args_.size()] = nullptr; Opt_.Reset(new Opt((int)Args_.size(), ArgsPtrs_.Get(), Format_.data())); - } - - inline ~TIterImpl() = default; - - inline void Next() { - OptLet_ = Opt_->Get(); - Arg_ = Opt_->Arg; - } - - inline char Key() const noexcept { - return (char)OptLet_; - } - - inline const char* Arg() const noexcept { - return Arg_; - } - - inline bool AtEnd() const noexcept { - return OptLet_ == EOF; - } - -private: - TVector<TString> Args_; - TArrayHolder<char*> ArgsPtrs_; - const TString Format_; - THolder<Opt> Opt_; - int OptLet_; - const char* Arg_; -}; - + } + + inline ~TIterImpl() = default; + + inline void Next() { + OptLet_ = Opt_->Get(); + Arg_ = Opt_->Arg; + } + + inline char Key() const noexcept { + return (char)OptLet_; + } + + inline const char* Arg() const noexcept { + return Arg_; + } + + inline bool AtEnd() const noexcept { + return OptLet_ == EOF; + } + +private: + TVector<TString> Args_; + TArrayHolder<char*> ArgsPtrs_; + const TString Format_; + THolder<Opt> Opt_; + int OptLet_; + const char* Arg_; +}; + TGetOpt::TIterator::TIterator() noexcept : Impl_(nullptr) -{ -} - -TGetOpt::TIterator::TIterator(const TGetOpt* parent) - : Impl_(new TIterImpl(parent)) -{ - Next(); -} - -void TGetOpt::TIterator::Next() { - Impl_->Next(); -} - +{ +} + +TGetOpt::TIterator::TIterator(const TGetOpt* parent) + : Impl_(new TIterImpl(parent)) +{ + Next(); +} + +void TGetOpt::TIterator::Next() { + Impl_->Next(); +} + char TGetOpt::TIterator::Key() const noexcept { - return Impl_->Key(); -} - + return Impl_->Key(); +} + bool TGetOpt::TIterator::AtEnd() const noexcept { - if (Impl_.Get()) { - return Impl_->AtEnd(); - } - - return true; -} - + if (Impl_.Get()) { + return Impl_->AtEnd(); + } + + return true; +} + const char* TGetOpt::TIterator::Arg() const noexcept { - if (Impl_.Get()) { - return Impl_->Arg(); - } - + if (Impl_.Get()) { + return Impl_->Arg(); + } + return nullptr; -} - +} + TGetOpt::TGetOpt(int argc, const char* const* argv, const TString& format) - : Impl_(new TImpl(argc, argv, format)) -{ -} + : Impl_(new TImpl(argc, argv, format)) +{ +} diff --git a/library/cpp/getopt/small/ygetopt.h b/library/cpp/getopt/small/ygetopt.h index 615d3dd18e..6f9b92cc79 100644 --- a/library/cpp/getopt/small/ygetopt.h +++ b/library/cpp/getopt/small/ygetopt.h @@ -1,72 +1,72 @@ #pragma once - + #include <util/generic/fwd.h> #include <util/generic/ptr.h> - -class TGetOpt { -public: - class TIterator { - friend class TGetOpt; - - public: - char Key() const noexcept; - const char* Arg() const noexcept; - - inline bool HaveArg() const noexcept { - return Arg(); - } - - inline void operator++() { - Next(); - } - - inline bool operator==(const TIterator& r) const noexcept { - return AtEnd() == r.AtEnd(); - } - - inline bool operator!=(const TIterator& r) const noexcept { - return !(*this == r); - } - - inline TIterator& operator*() noexcept { - return *this; - } - - inline const TIterator& operator*() const noexcept { - return *this; - } - - inline TIterator* operator->() noexcept { - return this; - } - - inline const TIterator* operator->() const noexcept { - return this; - } - - private: - TIterator() noexcept; - TIterator(const TGetOpt* parent); - - void Next(); - bool AtEnd() const noexcept; - - private: - class TIterImpl; - TSimpleIntrusivePtr<TIterImpl> Impl_; - }; - - TGetOpt(int argc, const char* const* argv, const TString& format); - - inline TIterator Begin() const { - return TIterator(this); - } - - inline TIterator End() const noexcept { - return TIterator(); - } - -private: - class TImpl; - TSimpleIntrusivePtr<TImpl> Impl_; -}; + +class TGetOpt { +public: + class TIterator { + friend class TGetOpt; + + public: + char Key() const noexcept; + const char* Arg() const noexcept; + + inline bool HaveArg() const noexcept { + return Arg(); + } + + inline void operator++() { + Next(); + } + + inline bool operator==(const TIterator& r) const noexcept { + return AtEnd() == r.AtEnd(); + } + + inline bool operator!=(const TIterator& r) const noexcept { + return !(*this == r); + } + + inline TIterator& operator*() noexcept { + return *this; + } + + inline const TIterator& operator*() const noexcept { + return *this; + } + + inline TIterator* operator->() noexcept { + return this; + } + + inline const TIterator* operator->() const noexcept { + return this; + } + + private: + TIterator() noexcept; + TIterator(const TGetOpt* parent); + + void Next(); + bool AtEnd() const noexcept; + + private: + class TIterImpl; + TSimpleIntrusivePtr<TIterImpl> Impl_; + }; + + TGetOpt(int argc, const char* const* argv, const TString& format); + + inline TIterator Begin() const { + return TIterator(this); + } + + inline TIterator End() const noexcept { + return TIterator(); + } + +private: + class TImpl; + TSimpleIntrusivePtr<TImpl> Impl_; +}; diff --git a/library/cpp/getopt/ut/last_getopt_ut.cpp b/library/cpp/getopt/ut/last_getopt_ut.cpp index c99a1d053d..0bdfff4f52 100644 --- a/library/cpp/getopt/ut/last_getopt_ut.cpp +++ b/library/cpp/getopt/ut/last_getopt_ut.cpp @@ -1,5 +1,5 @@ #include <library/cpp/getopt/last_getopt.h> - + #include <library/cpp/colorizer/colors.h> #include <library/cpp/testing/unittest/registar.h> @@ -11,14 +11,14 @@ using namespace NLastGetopt; namespace { - struct TOptsNoDefault: public TOpts { + struct TOptsNoDefault: public TOpts { TOptsNoDefault(const TStringBuf& optstring = TStringBuf()) : TOpts(optstring) { } }; - class TOptsParseResultTestWrapper: public TOptsParseResultException { + class TOptsParseResultTestWrapper: public TOptsParseResultException { TVector<const char*> Argv_; public: @@ -174,11 +174,11 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { UNIT_ASSERT_EXCEPTION( TOptsParseResultTestWrapper(&opts, V({"cp", "/etc", "/tmp/etc", "verbose", "nosymlink"})), - yexception); + yexception); UNIT_ASSERT_EXCEPTION( TOptsParseResultTestWrapper(&opts, V({"cp"})), - yexception); + yexception); opts.SetFreeArgsNum(2); TOptsParseResultTestWrapper r22(&opts, V({"cp", "/etc", "/var/tmp"})); @@ -347,16 +347,16 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { } Y_UNIT_TEST(TestDuplicatedOptionCrash) { - // this test is broken, cause UNIT_ASSERT(false) always throws - return; - + // this test is broken, cause UNIT_ASSERT(false) always throws + return; + bool exception = false; try { TOpts opts; opts.AddLongOption('x', "one"); opts.AddLongOption('x', "two"); UNIT_ASSERT(false); - } catch (...) { + } catch (...) { // we should go here, duplicating options are forbidden exception = true; } @@ -529,21 +529,21 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opt_d.Required(); UNIT_ASSERT_EXCEPTION( TOptsParseResultTestWrapper(&opts, V({"cmd"})), - TUsageException); + TUsageException); TOptsParseResultTestWrapper r3(&opts, V({"cmd", "-d11"})); UNIT_ASSERT_VALUES_EQUAL("11", r3.Get('d')); } - class HandlerStoreTrue { - bool* Flag; - + class HandlerStoreTrue { + bool* Flag; + public: - HandlerStoreTrue(bool* flag) + HandlerStoreTrue(bool* flag) : Flag(flag) { } - void operator()() { + void operator()() { *Flag = true; } }; @@ -561,11 +561,11 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { double fval = 0.0; opts.AddLongOption("flag1").RequiredArgument().StoreResult(&uval); opts.AddLongOption("flag2").RequiredArgument().StoreResultT<int>(&uval); - opts.AddLongOption("flag3").RequiredArgument().StoreMappedResult(&fval, (double (*)(double))fabs); - opts.AddLongOption("flag4").RequiredArgument().StoreMappedResult(&fval, (double (*)(double))sqrt); + opts.AddLongOption("flag3").RequiredArgument().StoreMappedResult(&fval, (double (*)(double))fabs); + opts.AddLongOption("flag4").RequiredArgument().StoreMappedResult(&fval, (double (*)(double))sqrt); UNIT_ASSERT_EXCEPTION( TOptsParseResultTestWrapper(&opts, V({"cmd", "--flag3", "-2.0", "--flag1", "-1"})), - yexception); + yexception); UNIT_ASSERT_VALUES_EQUAL(uval, 5u); UNIT_ASSERT_VALUES_EQUAL(fval, 2.0); TOptsParseResultTestWrapper r1(&opts, V({"cmd", "--flag4", "9.0", "--flag2", "-1"})); @@ -580,7 +580,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { TString title = TString("Sample ") + TString(prog).Quote() + " application"; opts.SetTitle(title); int argc = 2; - const char* cmd[] = {prog}; + const char* cmd[] = {prog}; TOptsParser parser(&opts, argc, cmd); TStringStream out; parser.PrintUsage(out); @@ -595,7 +595,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { const char* prog = "my_program"; TString customDescr = "<FILE|TABLE> USER [OPTIONS]"; int argc = 2; - const char* cmd[] = {prog}; + const char* cmd[] = {prog}; opts.SetCmdLineDescr(customDescr); TOptsParser parser(&opts, argc, cmd); TStringStream out; @@ -615,7 +615,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { opts.SetFreeArgTitle(0, "first_free_arg", "help"); opts.SetFreeArgTitle(2, "second_free_arg"); opts.AddSection("Section", "Section\n text"); - const char* cmd[] = {prog}; + const char* cmd[] = {prog}; TOptsParser parser(&opts, Y_ARRAY_SIZE(cmd), cmd); TStringStream out; NColorizer::TColors colors(true); @@ -663,12 +663,12 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { for (bool withColors : withColorsOpt) { TOpts opts; const char* prog = "my_program"; - opts.AddLongOption("option", "description 1").Required(); // long option - opts.AddLongOption('o', "other", "description 2"); // char and long option - opts.AddCharOption('d', "description 3").RequiredArgument("DD"); // char option + opts.AddLongOption("option", "description 1").Required(); // long option + opts.AddLongOption('o', "other", "description 2"); // char and long option + opts.AddCharOption('d', "description 3").RequiredArgument("DD"); // char option opts.AddCharOption('s', "description 4\ndescription 5\ndescription 6"); // multiline desc opts.AddLongOption('l', "very_very_very_loooong_ooooption", "description 7").RequiredArgument("LONG_ARGUMENT"); - const char* cmd[] = {prog}; + const char* cmd[] = {prog}; TOptsParser parser(&opts, Y_ARRAY_SIZE(cmd), cmd); TStringStream out; @@ -730,7 +730,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { TStringBuilder keyvals; TOptsNoDefault opts; - opts.AddLongOption("set").KVHandler([&keyvals](TString k, TString v) { keyvals << k << ":" << v << ","; }); + opts.AddLongOption("set").KVHandler([&keyvals](TString k, TString v) { keyvals << k << ":" << v << ","; }); TOptsParseResultTestWrapper r(&opts, V({"cmd", "--set", "x=1", "--set", "y=2", "--set=z=3"})); @@ -740,7 +740,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { Y_UNIT_TEST(TestEasySetup) { TEasySetup opts; bool flag = false; - opts('v', "version", "print version information")('a', "abstract", "some abstract param", true)('b', "buffer", "SIZE", "some param with argument")('c', "count", "SIZE", "some param with required argument")('t', "true", HandlerStoreTrue(&flag), "Some arg with handler")("global", SimpleHander, "Another arg with handler"); + opts('v', "version", "print version information")('a', "abstract", "some abstract param", true)('b', "buffer", "SIZE", "some param with argument")('c', "count", "SIZE", "some param with required argument")('t', "true", HandlerStoreTrue(&flag), "Some arg with handler")("global", SimpleHander, "Another arg with handler"); { gSimpleFlag = false; @@ -758,7 +758,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { { UNIT_ASSERT_EXCEPTION( TOptsParseResultTestWrapper(&opts, V({"cmd", "--true"})), - TUsageException); + TUsageException); } { @@ -777,7 +777,7 @@ Y_UNIT_TEST_SUITE(TLastGetoptTests) { // Should throw TUsageException. Other exception types, no exceptions at all and exit(1) are failures UNIT_ASSERT_EXCEPTION( TOptsParseResultException(&opts, Y_ARRAY_SIZE(argv), argv), - TUsageException); + TUsageException); } Y_UNIT_TEST(TestFreeArgsStoreResult) { diff --git a/library/cpp/getopt/ut/opt2_ut.cpp b/library/cpp/getopt/ut/opt2_ut.cpp index 0e7464747c..4f7b615037 100644 --- a/library/cpp/getopt/ut/opt2_ut.cpp +++ b/library/cpp/getopt/ut/opt2_ut.cpp @@ -1,5 +1,5 @@ #include <library/cpp/getopt/opt2.h> - + #include <library/cpp/testing/unittest/registar.h> //using namespace NLastGetopt; @@ -8,25 +8,25 @@ Y_UNIT_TEST_SUITE(Opt2Test) { Y_UNIT_TEST(TestSimple) { int argc = 8; char* argv[] = { - (char*)"cmd", - (char*)"--aaaa=aaaa", - (char*)"zz", - (char*)"-x1", - (char*)"-x2", - (char*)"-c", - (char*)"-d8", - (char*)"ww", + (char*)"cmd", + (char*)"--aaaa=aaaa", + (char*)"zz", + (char*)"-x1", + (char*)"-x2", + (char*)"-c", + (char*)"-d8", + (char*)"ww", }; Opt2 opt(argc, argv, "A:b:cd:e:x:", 2, "aaaa=A"); - const char* edef = "edef"; - const char* a = opt.Arg('A', "<var_name> - usage of -A"); - int b = opt.Int('b', "<var_name> - usage of -b", 2); - bool c = opt.Has('c', "usage of -c"); - int d = opt.Int('d', "<var_name> - usage of -d", 13); - const char* e = opt.Arg('e', "<unused> - only default is really used", edef); - const TVector<const char*>& x = opt.MArg('x', "<var_name> - usage of -x"); + const char* edef = "edef"; + const char* a = opt.Arg('A', "<var_name> - usage of -A"); + int b = opt.Int('b', "<var_name> - usage of -b", 2); + bool c = opt.Has('c', "usage of -c"); + int d = opt.Int('d', "<var_name> - usage of -d", 13); + const char* e = opt.Arg('e', "<unused> - only default is really used", edef); + const TVector<const char*>& x = opt.MArg('x', "<var_name> - usage of -x"); UNIT_ASSERT(!opt.AutoUsage("<L> <M>")); UNIT_ASSERT_VALUES_EQUAL("aaaa", a); @@ -46,17 +46,17 @@ Y_UNIT_TEST_SUITE(Opt2Test) { Y_UNIT_TEST(TestErrors1) { int argc = 4; char* argv[] = { - (char*)"cmd", - (char*)"zz", - (char*)"-c", - (char*)"-e", + (char*)"cmd", + (char*)"zz", + (char*)"-c", + (char*)"-e", }; Opt2 opt(argc, argv, "ce:", 2); - const char* edef = "edef"; - bool c = opt.Has('c', "usage of -c"); - const char* e = opt.Arg('e', "<unused> - only default is really used", edef); + const char* edef = "edef"; + bool c = opt.Has('c', "usage of -c"); + const char* e = opt.Arg('e', "<unused> - only default is really used", edef); UNIT_ASSERT(c); UNIT_ASSERT_VALUES_EQUAL((void*)edef, e); } diff --git a/library/cpp/getopt/ut/opt_ut.cpp b/library/cpp/getopt/ut/opt_ut.cpp index 441aa493a0..031105b4cb 100644 --- a/library/cpp/getopt/ut/opt_ut.cpp +++ b/library/cpp/getopt/ut/opt_ut.cpp @@ -1,5 +1,5 @@ #include <library/cpp/getopt/opt.h> - + #include <library/cpp/testing/unittest/registar.h> #include <util/string/vector.h> @@ -7,7 +7,7 @@ Y_UNIT_TEST_SUITE(OptTest) { Y_UNIT_TEST(TestSimple) { int argc = 3; char* argv[] = { - (char*)"cmd", (char*)"-x"}; + (char*)"cmd", (char*)"-x"}; Opt opt(argc, argv, ""); opt.Err = false; // be quiet UNIT_ASSERT_VALUES_EQUAL('?', opt.Get()); @@ -18,9 +18,9 @@ Y_UNIT_TEST_SUITE(OptTest) { Y_UNIT_TEST(TestFreeArguments) { Opt::Ion options[] = { - {"some-option", Opt::WithArg, nullptr, 123}, - {nullptr, Opt::WithoutArg, nullptr, 0}}; - const char* argv[] = {"cmd", "ARG1", "-some-option", "ARG2", "ARG3", nullptr}; + {"some-option", Opt::WithArg, nullptr, 123}, + {nullptr, Opt::WithoutArg, nullptr, 0}}; + const char* argv[] = {"cmd", "ARG1", "-some-option", "ARG2", "ARG3", nullptr}; int argc = 5; Opt opts(argc, argv, "", options); @@ -30,10 +30,10 @@ Y_UNIT_TEST_SUITE(OptTest) { Y_UNIT_TEST(TestLongOption) { const int SOME_OPTION_ID = 12345678; Opt::Ion options[] = { - {"some-option", Opt::WithArg, nullptr, SOME_OPTION_ID}, - {nullptr, Opt::WithoutArg, nullptr, 0}}; + {"some-option", Opt::WithArg, nullptr, SOME_OPTION_ID}, + {nullptr, Opt::WithoutArg, nullptr, 0}}; for (int doubleDash = 0; doubleDash <= 1; ++doubleDash) { - const char* argv[] = {"cmd", "ARG1", (doubleDash ? "--some-option" : "-some-option"), "ARG2", "ARG3", nullptr}; + const char* argv[] = {"cmd", "ARG1", (doubleDash ? "--some-option" : "-some-option"), "ARG2", "ARG3", nullptr}; int argc = 5; Opt opts(argc, argv, "", options); @@ -42,7 +42,7 @@ Y_UNIT_TEST_SUITE(OptTest) { while ((optlet = opts.Get()) != EOF) { if (optlet == SOME_OPTION_ID) { optionValue = opts.GetArg(); - } else { + } else { UNIT_FAIL("don't expected any options, except -some-option"); } } diff --git a/library/cpp/getopt/ut/posix_getopt_ut.cpp b/library/cpp/getopt/ut/posix_getopt_ut.cpp index b6d374bf28..549a5ae3a2 100644 --- a/library/cpp/getopt/ut/posix_getopt_ut.cpp +++ b/library/cpp/getopt/ut/posix_getopt_ut.cpp @@ -1,5 +1,5 @@ #include <library/cpp/getopt/posix_getopt.h> - + #include <library/cpp/testing/unittest/registar.h> using namespace NLastGetopt; @@ -7,8 +7,8 @@ using namespace NLastGetopt; Y_UNIT_TEST_SUITE(TPosixGetoptTest) { Y_UNIT_TEST(TestSimple) { int argc = 6; - const char* argv0[] = {"program", "-b", "-f1", "-f", "2", "zzzz"}; - char** const argv = (char**)argv0; + const char* argv0[] = {"program", "-b", "-f1", "-f", "2", "zzzz"}; + char** const argv = (char**)argv0; NLastGetopt::optreset = 1; UNIT_ASSERT_VALUES_EQUAL('b', NLastGetopt::getopt(argc, argv, "bf:")); @@ -25,14 +25,14 @@ Y_UNIT_TEST_SUITE(TPosixGetoptTest) { int daggerset = 0; /* options descriptor */ const NLastGetopt::option longopts[] = { - {"buffy", no_argument, nullptr, 'b'}, - {"fluoride", required_argument, nullptr, 'f'}, - {"daggerset", no_argument, &daggerset, 1}, - {nullptr, 0, nullptr, 0}}; + {"buffy", no_argument, nullptr, 'b'}, + {"fluoride", required_argument, nullptr, 'f'}, + {"daggerset", no_argument, &daggerset, 1}, + {nullptr, 0, nullptr, 0}}; int argc = 7; - const char* argv0[] = {"program", "-b", "--buffy", "-f1", "--fluoride=2", "--daggerset", "zzzz"}; - char** const argv = (char**)argv0; + const char* argv0[] = {"program", "-b", "--buffy", "-f1", "--fluoride=2", "--daggerset", "zzzz"}; + char** const argv = (char**)argv0; int longIndex; @@ -53,14 +53,14 @@ Y_UNIT_TEST_SUITE(TPosixGetoptTest) { int daggerset = 0; /* options descriptor */ const NLastGetopt::option longopts[] = { - {"buffy", no_argument, nullptr, 'b'}, - {"fluoride", required_argument, nullptr, 'f'}, - {"daggerset", no_argument, &daggerset, 1}, - {nullptr, 0, nullptr, 0}}; + {"buffy", no_argument, nullptr, 'b'}, + {"fluoride", required_argument, nullptr, 'f'}, + {"daggerset", no_argument, &daggerset, 1}, + {nullptr, 0, nullptr, 0}}; int argc = 7; - const char* argv0[] = {"program", "aa", "-b", "bb", "cc", "--buffy", "dd"}; - char** const argv = (char**)argv0; + const char* argv0[] = {"program", "aa", "-b", "bb", "cc", "--buffy", "dd"}; + char** const argv = (char**)argv0; NLastGetopt::optreset = 1; UNIT_ASSERT_VALUES_EQUAL('b', NLastGetopt::getopt_long(argc, argv, "bf:", longopts, nullptr)); @@ -72,13 +72,13 @@ Y_UNIT_TEST_SUITE(TPosixGetoptTest) { Y_UNIT_TEST(TestNoOptionsOptionsWithDoubleDash) { const NLastGetopt::option longopts[] = { - {"buffy", no_argument, nullptr, 'b'}, - {"fluoride", no_argument, nullptr, 'f'}, - {nullptr, 0, nullptr, 0}}; + {"buffy", no_argument, nullptr, 'b'}, + {"fluoride", no_argument, nullptr, 'f'}, + {nullptr, 0, nullptr, 0}}; int argc = 2; - const char* argv0[] = {"program", "--bf"}; - char** const argv = (char**)argv0; + const char* argv0[] = {"program", "--bf"}; + char** const argv = (char**)argv0; NLastGetopt::optreset = 1; UNIT_ASSERT_VALUES_EQUAL('?', NLastGetopt::getopt_long(argc, argv, "bf", longopts, nullptr)); @@ -86,32 +86,32 @@ Y_UNIT_TEST_SUITE(TPosixGetoptTest) { Y_UNIT_TEST(TestLongOnly) { const NLastGetopt::option longopts[] = { - {"foo", no_argument, nullptr, 'F'}, - {"fluoride", no_argument, nullptr, 'f'}, - {"ogogo", no_argument, nullptr, 'o'}, - {nullptr, 0, nullptr, 0}}; + {"foo", no_argument, nullptr, 'F'}, + {"fluoride", no_argument, nullptr, 'f'}, + {"ogogo", no_argument, nullptr, 'o'}, + {nullptr, 0, nullptr, 0}}; int argc = 4; - const char* argv0[] = {"program", "--foo", "-foo", "-fo"}; - char** const argv = (char**)argv0; + const char* argv0[] = {"program", "--foo", "-foo", "-fo"}; + char** const argv = (char**)argv0; NLastGetopt::optreset = 1; UNIT_ASSERT_VALUES_EQUAL('F', NLastGetopt::getopt_long_only(argc, argv, "fo", longopts, nullptr)); UNIT_ASSERT_VALUES_EQUAL('F', NLastGetopt::getopt_long_only(argc, argv, "fo", longopts, nullptr)); UNIT_ASSERT_VALUES_EQUAL('f', NLastGetopt::getopt_long_only(argc, argv, "fo", longopts, nullptr)); UNIT_ASSERT_VALUES_EQUAL('o', NLastGetopt::getopt_long_only(argc, argv, "fo", longopts, nullptr)); - UNIT_ASSERT_VALUES_EQUAL(-1, NLastGetopt::getopt_long_only(argc, argv, "fo", longopts, nullptr)); + UNIT_ASSERT_VALUES_EQUAL(-1, NLastGetopt::getopt_long_only(argc, argv, "fo", longopts, nullptr)); } Y_UNIT_TEST(TestLongWithoutOnlySingleDashNowAllowed) { const NLastGetopt::option longopts[] = { - {"foo", no_argument, nullptr, 'F'}, - {"zoo", no_argument, nullptr, 'z'}, - {nullptr, 0, nullptr, 0}}; + {"foo", no_argument, nullptr, 'F'}, + {"zoo", no_argument, nullptr, 'z'}, + {nullptr, 0, nullptr, 0}}; int argc = 2; - const char* argv0[] = {"program", "-foo"}; - char** const argv = (char**)argv0; + const char* argv0[] = {"program", "-foo"}; + char** const argv = (char**)argv0; NLastGetopt::optreset = 1; UNIT_ASSERT_VALUES_EQUAL('?', NLastGetopt::getopt_long(argc, argv, "z", longopts, nullptr)); diff --git a/library/cpp/getopt/ut/ygetopt_ut.cpp b/library/cpp/getopt/ut/ygetopt_ut.cpp index a76f117216..99fa51281e 100644 --- a/library/cpp/getopt/ut/ygetopt_ut.cpp +++ b/library/cpp/getopt/ut/ygetopt_ut.cpp @@ -1,45 +1,45 @@ #include <library/cpp/getopt/ygetopt.h> - + #include <library/cpp/testing/unittest/registar.h> - -class TGetOptTest: public TTestBase { - UNIT_TEST_SUITE(TGetOptTest); - UNIT_TEST(TestGetOpt); - UNIT_TEST_EXCEPTION(TestZeroArgC, yexception); - UNIT_TEST_SUITE_END(); - -public: - void TestGetOpt(); - void TestZeroArgC(); -}; - -UNIT_TEST_SUITE_REGISTRATION(TGetOptTest); - -void TGetOptTest::TestZeroArgC() { + +class TGetOptTest: public TTestBase { + UNIT_TEST_SUITE(TGetOptTest); + UNIT_TEST(TestGetOpt); + UNIT_TEST_EXCEPTION(TestZeroArgC, yexception); + UNIT_TEST_SUITE_END(); + +public: + void TestGetOpt(); + void TestZeroArgC(); +}; + +UNIT_TEST_SUITE_REGISTRATION(TGetOptTest); + +void TGetOptTest::TestZeroArgC() { TGetOpt opt(0, nullptr, ""); -} - -void TGetOptTest::TestGetOpt() { - const char* argv[] = { - "/usr/bin/bash", - "-f", - "-p", - "qwerty123", - "-z", - "-q", - nullptr}; - +} + +void TGetOptTest::TestGetOpt() { + const char* argv[] = { + "/usr/bin/bash", + "-f", + "-p", + "qwerty123", + "-z", + "-q", + nullptr}; + TString res; const TString format = "qzp:f"; - TGetOpt opt(sizeof(argv) / sizeof(*argv) - 1, argv, format); - - for (TGetOpt::TIterator it = opt.Begin(); it != opt.End(); ++it) { - res += it->Key(); - - if (it->HaveArg()) { - res += it->Arg(); - } - } - - UNIT_ASSERT_EQUAL(res, "fpqwerty123zq"); -} + TGetOpt opt(sizeof(argv) / sizeof(*argv) - 1, argv, format); + + for (TGetOpt::TIterator it = opt.Begin(); it != opt.End(); ++it) { + res += it->Key(); + + if (it->HaveArg()) { + res += it->Arg(); + } + } + + UNIT_ASSERT_EQUAL(res, "fpqwerty123zq"); +} diff --git a/library/cpp/getopt/ya.make b/library/cpp/getopt/ya.make index 6df23b22b2..18adcd6051 100644 --- a/library/cpp/getopt/ya.make +++ b/library/cpp/getopt/ya.make @@ -1,4 +1,4 @@ -LIBRARY() +LIBRARY() OWNER(pg) diff --git a/library/cpp/getopt/ygetopt.h b/library/cpp/getopt/ygetopt.h index fd018b3128..559a24bfe8 100644 --- a/library/cpp/getopt/ygetopt.h +++ b/library/cpp/getopt/ygetopt.h @@ -1,3 +1,3 @@ #pragma once - + #include <library/cpp/getopt/small/ygetopt.h> diff --git a/library/cpp/histogram/adaptive/adaptive_histogram.cpp b/library/cpp/histogram/adaptive/adaptive_histogram.cpp index cbfc494021..4c41575bad 100644 --- a/library/cpp/histogram/adaptive/adaptive_histogram.cpp +++ b/library/cpp/histogram/adaptive/adaptive_histogram.cpp @@ -78,8 +78,8 @@ namespace NKiwiAggr { if (histo.GetType() == HT_ADAPTIVE_DISTANCE_HISTOGRAM || histo.GetType() == HT_ADAPTIVE_WEIGHT_HISTOGRAM || histo.GetType() == HT_ADAPTIVE_WARD_HISTOGRAM || - histo.GetType() == HT_ADAPTIVE_HISTOGRAM) - { + histo.GetType() == HT_ADAPTIVE_HISTOGRAM) + { Y_VERIFY(histo.FreqSize() == histo.PositionSize(), "Corrupted histo"); for (size_t j = 0; j < histo.FreqSize(); ++j) { double value = histo.GetPosition(j); @@ -130,7 +130,7 @@ namespace NKiwiAggr { if (adaptiveHisto) { histogramsToMergeRepacked.push_back(histogramsToMerge[i]); } else { - histogramsToMergeRepacked.push_back(IHistogramPtr(new TAdaptiveHistogram(histogramsToMerge[i].Get(), Intervals, Id, CalcQuality))); // Convert histograms that are not of TFixedBinHistogram type + histogramsToMergeRepacked.push_back(IHistogramPtr(new TAdaptiveHistogram(histogramsToMerge[i].Get(), Intervals, Id, CalcQuality))); // Convert histograms that are not of TFixedBinHistogram type } if (histogramsToMergeRepacked.back()->Empty()) { continue; @@ -180,8 +180,8 @@ namespace NKiwiAggr { } void TAdaptiveHistogram::FromProto(const THistogram& histo) { - Y_VERIFY(histo.HasType(), "Attempt to parse TAdaptiveHistogram from THistogram protobuf with no Type field set"); - ; + Y_VERIFY(histo.HasType(), "Attempt to parse TAdaptiveHistogram from THistogram protobuf with no Type field set"); + ; switch (histo.GetType()) { // check that histogram type could be deduced case HT_ADAPTIVE_DISTANCE_HISTOGRAM: case HT_ADAPTIVE_WEIGHT_HISTOGRAM: @@ -191,7 +191,7 @@ namespace NKiwiAggr { if (CalcQuality != nullptr) break; // ok [[fallthrough]]; - default: // not ok + default: // not ok ythrow yexception() << "Attempt to parse TAdaptiveHistogram from THistogram protobuf record of type = " << (ui32)histo.GetType(); } @@ -300,7 +300,7 @@ namespace NKiwiAggr { bound, PrecomputedBins, LowerBound(PrecomputedBins.begin(), PrecomputedBins.end(), TFastBin{bound, -1.0, 0, 0}), - [](const auto& it) { return it->SumAbove; }); + [](const auto& it) { return it->SumAbove; }); } else { return GetSumAboveBoundImpl( bound, @@ -313,7 +313,7 @@ namespace NKiwiAggr { sum += it->second; } return sum; - }); + }); } } @@ -333,7 +333,7 @@ namespace NKiwiAggr { bound, PrecomputedBins, LowerBound(PrecomputedBins.begin(), PrecomputedBins.end(), TFastBin{bound, -1.0, 0, 0}), - [](const auto& it) { return it->SumBelow; }); + [](const auto& it) { return it->SumBelow; }); } else { return GetSumBelowBoundImpl( bound, @@ -345,7 +345,7 @@ namespace NKiwiAggr { sum += it->second; } return sum; - }); + }); } } @@ -634,4 +634,4 @@ namespace NKiwiAggr { return sum; } -} +} diff --git a/library/cpp/histogram/adaptive/adaptive_histogram.h b/library/cpp/histogram/adaptive/adaptive_histogram.h index fa8f48433f..694c198d11 100644 --- a/library/cpp/histogram/adaptive/adaptive_histogram.h +++ b/library/cpp/histogram/adaptive/adaptive_histogram.h @@ -10,7 +10,7 @@ #include <util/generic/vector.h> namespace NKiwiAggr { - class TAdaptiveHistogram: private TNonCopyable, public IHistogram { + class TAdaptiveHistogram: private TNonCopyable, public IHistogram { protected: static const size_t DEFAULT_INTERVALS = 100; @@ -25,7 +25,7 @@ namespace NKiwiAggr { double SumBelow; double SumAbove; - TFastBin(double first_, double second_, double sumBelow = 0, double sumAbove = 0) + TFastBin(double first_, double second_, double sumBelow = 0, double sumAbove = 0) : first(first_) , second(second_) , SumBelow(sumBelow) @@ -33,7 +33,7 @@ namespace NKiwiAggr { { } - bool operator<(const TFastBin& rhs) const { + bool operator<(const TFastBin& rhs) const { return first < rhs.first; } }; @@ -102,7 +102,7 @@ namespace NKiwiAggr { double GetSumBelowBoundImpl(double bound, const TBins& bins, typename TBins::const_iterator rightBin, const TGetSumBelow& getSumBelow) const; }; - template <TQualityFunction QualityFunction> + template <TQualityFunction QualityFunction> class TDefinedAdaptiveHistogram: public TAdaptiveHistogram { public: TDefinedAdaptiveHistogram(size_t intervals, ui64 id = 0) @@ -128,4 +128,4 @@ namespace NKiwiAggr { typedef TDefinedAdaptiveHistogram<CalcWeightQuality> TAdaptiveWeightHistogram; typedef TDefinedAdaptiveHistogram<CalcWardQuality> TAdaptiveWardHistogram; -} +} diff --git a/library/cpp/histogram/adaptive/auto_histogram.h b/library/cpp/histogram/adaptive/auto_histogram.h index 9fdf0b9abe..a7a524d7ef 100644 --- a/library/cpp/histogram/adaptive/auto_histogram.h +++ b/library/cpp/histogram/adaptive/auto_histogram.h @@ -10,7 +10,7 @@ #include <util/generic/yexception.h> namespace NKiwiAggr { - class TAutoHistogram: private TNonCopyable, public IHistogram { + class TAutoHistogram: private TNonCopyable, public IHistogram { private: static const size_t DEFAULT_INTERVALS = 100; @@ -39,7 +39,7 @@ namespace NKiwiAggr { ythrow yexception() << "IHistogram constructor is not defined for TAutoHistogram"; } - virtual ~TAutoHistogram() { + virtual ~TAutoHistogram() { } virtual void Clear() { @@ -145,4 +145,4 @@ namespace NKiwiAggr { } }; -} +} diff --git a/library/cpp/histogram/adaptive/block_histogram.cpp b/library/cpp/histogram/adaptive/block_histogram.cpp index 6586d13ff6..0920bf435b 100644 --- a/library/cpp/histogram/adaptive/block_histogram.cpp +++ b/library/cpp/histogram/adaptive/block_histogram.cpp @@ -52,7 +52,7 @@ namespace { private: void SwapPositions(ui32 x, ui32 y) { - std::swap(A[x], A[y]); + std::swap(A[x], A[y]); Pos[A[x]] = x; Pos[A[y]] = y; } @@ -147,8 +147,8 @@ namespace NKiwiAggr { if (histo.GetType() == HT_ADAPTIVE_DISTANCE_HISTOGRAM || histo.GetType() == HT_ADAPTIVE_WEIGHT_HISTOGRAM || histo.GetType() == HT_ADAPTIVE_WARD_HISTOGRAM || - histo.GetType() == HT_ADAPTIVE_HISTOGRAM) - { + histo.GetType() == HT_ADAPTIVE_HISTOGRAM) + { Y_VERIFY(histo.FreqSize() == histo.PositionSize(), "Corrupted histo"); for (size_t j = 0; j < histo.FreqSize(); ++j) { double value = histo.GetPosition(j); @@ -188,7 +188,7 @@ namespace NKiwiAggr { } } - void TBlockHistogram::Merge(TVector<IHistogramPtr> histogramsToMerge) { + void TBlockHistogram::Merge(TVector<IHistogramPtr> histogramsToMerge) { Y_UNUSED(histogramsToMerge); ythrow yexception() << "IHistogram::Merge(TVector<IHistogramPtr>) is not defined for TBlockHistogram"; } @@ -204,15 +204,15 @@ namespace NKiwiAggr { } void TBlockHistogram::FromProto(const THistogram& histo) { - Y_VERIFY(histo.HasType(), "Attempt to parse TBlockHistogram from THistogram protobuf with no Type field set"); - ; + Y_VERIFY(histo.HasType(), "Attempt to parse TBlockHistogram from THistogram protobuf with no Type field set"); + ; switch (histo.GetType()) { // check that histogram type is correct case HT_ADAPTIVE_DISTANCE_HISTOGRAM: case HT_ADAPTIVE_WEIGHT_HISTOGRAM: case HT_ADAPTIVE_WARD_HISTOGRAM: case HT_ADAPTIVE_HISTOGRAM: break; // ok - default: // not ok + default: // not ok ythrow yexception() << "Attempt to parse TBlockHistogram from THistogram protobuf record of type = " << (ui32)histo.GetType(); } @@ -312,7 +312,7 @@ namespace NKiwiAggr { Sort(Bins.begin() + PrevSize, Bins.end()); if (PrevSize != 0) { TVector<TWeightedValue> temp(Bins.begin(), Bins.begin() + PrevSize); - std::merge(temp.begin(), temp.end(), Bins.begin() + PrevSize, Bins.end(), Bins.begin()); + std::merge(temp.begin(), temp.end(), Bins.begin() + PrevSize, Bins.end(), Bins.begin()); } } @@ -496,13 +496,13 @@ namespace NKiwiAggr { TBlockWardHistogram::TBlockWardHistogram(size_t intervals, ui64 id, size_t shrinkSize) : TBlockHistogram(HT_ADAPTIVE_WARD_HISTOGRAM, CalcWardQuality, intervals, id, shrinkSize) - { - } + { + } bool TBlockWardHistogram::CalcSplitInfo( const TCumulatives::const_iterator beg, const TCumulatives::const_iterator end, // (!) points to the final element - TSplitInfo& splitInfo // out + TSplitInfo& splitInfo // out ) { if (end - beg < 2) { return false; @@ -543,7 +543,7 @@ namespace NKiwiAggr { TCumulative cumulative = {0., 0.}; cumulatives.push_back(cumulative); for (size_t i = 0; i < Bins.size(); i++) { - cumulative.first += Bins[i].second; + cumulative.first += Bins[i].second; cumulative.second += Bins[i].second * Bins[i].first; cumulatives.push_back(cumulative); } @@ -584,10 +584,10 @@ namespace NKiwiAggr { auto splitBeg = *it; auto splitEnd = *(it + 1); double cnt = (splitEnd->first - splitBeg->first); - double mu = (splitEnd->second - splitBeg->second) / cnt; + double mu = (splitEnd->second - splitBeg->second) / cnt; Bins.push_back(TWeightedValue(mu, cnt)); } } -} +} diff --git a/library/cpp/histogram/adaptive/block_histogram.h b/library/cpp/histogram/adaptive/block_histogram.h index 266bb2f2b2..a302f91aef 100644 --- a/library/cpp/histogram/adaptive/block_histogram.h +++ b/library/cpp/histogram/adaptive/block_histogram.h @@ -27,12 +27,12 @@ namespace NKiwiAggr { */ class TBlockHistogram: private TNonCopyable, public IHistogram { protected: - static const size_t SHRINK_MULTIPLIER = 2; + static const size_t SHRINK_MULTIPLIER = 2; static const size_t GREEDY_SHRINK_MULTIPLIER = 4; - static const size_t DEFAULT_INTERVALS = 100; + static const size_t DEFAULT_INTERVALS = 100; static const size_t DEFAULT_SHRINK_SIZE = DEFAULT_INTERVALS * (SHRINK_MULTIPLIER + GREEDY_SHRINK_MULTIPLIER); - const EHistogramType Type; + const EHistogramType Type; const TQualityFunction CalcQuality; size_t Intervals; @@ -51,7 +51,7 @@ namespace NKiwiAggr { TBlockHistogram(EHistogramType type, TQualityFunction calcQuality, size_t intervals, ui64 id = 0, size_t shrinkSize = DEFAULT_SHRINK_SIZE); - virtual ~TBlockHistogram() { + virtual ~TBlockHistogram() { } virtual void Clear(); @@ -61,7 +61,7 @@ namespace NKiwiAggr { virtual void Merge(const THistogram& histo, double multiplier); virtual void Merge(const TVector<THistogram>& histogramsToMerge); - virtual void Merge(TVector<IHistogramPtr> histogramsToMerge); // not implemented + virtual void Merge(TVector<IHistogramPtr> histogramsToMerge); // not implemented virtual void Multiply(double factor); @@ -99,12 +99,12 @@ namespace NKiwiAggr { // TBlockWeightHistogram ///////////////////////// - class TBlockWeightHistogram: public TBlockHistogram { + class TBlockWeightHistogram: public TBlockHistogram { public: TBlockWeightHistogram(size_t intervals, ui64 id = 0, size_t shrinkSize = DEFAULT_SHRINK_SIZE); - virtual ~TBlockWeightHistogram() { - } + virtual ~TBlockWeightHistogram() { + } private: virtual void FastGreedyShrink(size_t intervals) final; @@ -114,15 +114,15 @@ namespace NKiwiAggr { // TBlockWardHistogram /////////////////////// - class TBlockWardHistogram: public TBlockHistogram { + class TBlockWardHistogram: public TBlockHistogram { public: TBlockWardHistogram(size_t intervals, ui64 id = 0, size_t shrinkSize = DEFAULT_SHRINK_SIZE); - virtual ~TBlockWardHistogram() { - } + virtual ~TBlockWardHistogram() { + } private: - using TCumulative = std::pair<double, double>; // cumulative sum of (weights, weighted centers) + using TCumulative = std::pair<double, double>; // cumulative sum of (weights, weighted centers) using TCumulatives = TVector<TCumulative>; struct TSplitInfo { @@ -145,4 +145,4 @@ namespace NKiwiAggr { TSplitInfo& splitInfo); }; -} +} diff --git a/library/cpp/histogram/adaptive/common.cpp b/library/cpp/histogram/adaptive/common.cpp index afc6322fce..f79a8f2e74 100644 --- a/library/cpp/histogram/adaptive/common.cpp +++ b/library/cpp/histogram/adaptive/common.cpp @@ -10,8 +10,8 @@ namespace NKiwiAggr { } TWeightedValue CalcWardQuality(const TWeightedValue& left, const TWeightedValue& right) { - const double N1 = left.second; - const double N2 = right.second; + const double N1 = left.second; + const double N2 = right.second; const double mu1 = left.first; const double mu2 = right.first; return TWeightedValue(N1 * N2 / (N1 + N2) * (mu1 - mu2) * (mu1 - mu2), left.first); diff --git a/library/cpp/histogram/adaptive/common.h b/library/cpp/histogram/adaptive/common.h index c0f7dfb26b..af71023e1b 100644 --- a/library/cpp/histogram/adaptive/common.h +++ b/library/cpp/histogram/adaptive/common.h @@ -3,7 +3,7 @@ #include <utility> namespace NKiwiAggr { - using TWeightedValue = std::pair<double, double>; // value, weight + using TWeightedValue = std::pair<double, double>; // value, weight using TQualityFunction = TWeightedValue (*)(const TWeightedValue&, const TWeightedValue&); TWeightedValue CalcDistanceQuality(const TWeightedValue& left, const TWeightedValue& right); diff --git a/library/cpp/histogram/adaptive/fixed_bin_histogram.cpp b/library/cpp/histogram/adaptive/fixed_bin_histogram.cpp index 558aba9e2d..b92da24809 100644 --- a/library/cpp/histogram/adaptive/fixed_bin_histogram.cpp +++ b/library/cpp/histogram/adaptive/fixed_bin_histogram.cpp @@ -156,7 +156,7 @@ namespace NKiwiAggr { fixedBinHisto->Initialize(); histogramsToMergeRepacked.push_back(histogramsToMerge[i]); } else { - histogramsToMergeRepacked.push_back(IHistogramPtr(new TFixedBinHistogram(histogramsToMerge[i].Get(), Intervals, Id, TrainingSetSize))); // Convert histograms that are not of TFixedBinHistogram type + histogramsToMergeRepacked.push_back(IHistogramPtr(new TFixedBinHistogram(histogramsToMerge[i].Get(), Intervals, Id, TrainingSetSize))); // Convert histograms that are not of TFixedBinHistogram type } histograms.push_back(dynamic_cast<TFixedBinHistogram*>(histogramsToMergeRepacked.back().Get())); } @@ -213,7 +213,7 @@ namespace NKiwiAggr { Initialize(); } Sum *= factor; - for (i32 i = FirstUsedBin; i <= LastUsedBin; ++i) { + for (i32 i = FirstUsedBin; i <= LastUsedBin; ++i) { Freqs[i] *= factor; } } @@ -325,7 +325,7 @@ namespace NKiwiAggr { if (IsEmpty) { return 0.0; } - if (BinRange == 0.0) { // special case - all values added to histogram are the same + if (BinRange == 0.0) { // special case - all values added to histogram are the same return (bound <= ReferencePoint) ? Sum : 0.0; } i32 bin = CalcBin(bound); @@ -337,7 +337,7 @@ namespace NKiwiAggr { } double binStart = BinStart(bin); double binEnd = BinEnd(bin); - double result = (bound < binStart) ? Freqs[bin] : Freqs[bin] * (binEnd - bound) / (binEnd - binStart); + double result = (bound < binStart) ? Freqs[bin] : Freqs[bin] * (binEnd - bound) / (binEnd - binStart); for (i32 i = bin + 1; i <= LastUsedBin; ++i) { result += Freqs[i]; } @@ -351,7 +351,7 @@ namespace NKiwiAggr { if (IsEmpty) { return 0.0; } - if (BinRange == 0.0) { // special case - all values added to histogram are the same + if (BinRange == 0.0) { // special case - all values added to histogram are the same return (bound > ReferencePoint) ? Sum : 0.0; } i32 bin = CalcBin(bound); @@ -363,7 +363,7 @@ namespace NKiwiAggr { } double binStart = BinStart(bin); double binEnd = BinEnd(bin); - double result = (bound > binEnd) ? Freqs[bin] : Freqs[bin] * (bound - binStart) / (binEnd - binStart); + double result = (bound > binEnd) ? Freqs[bin] : Freqs[bin] * (bound - binStart) / (binEnd - binStart); for (i32 i = bin - 1; i >= FirstUsedBin; --i) { result += Freqs[i]; } @@ -516,7 +516,7 @@ namespace NKiwiAggr { memset(&(ReserveFreqs[0]), 0, ReserveFreqs.size() * sizeof(double)); double newBinRange = CalcBinRange(newReferencePoint, newMaxValue); - for (i32 i = FirstUsedBin; i <= LastUsedBin; ++i) { + for (i32 i = FirstUsedBin; i <= LastUsedBin; ++i) { double binStart = BinStart(i); double binEnd = BinEnd(i); double freq = Freqs[i]; @@ -535,4 +535,4 @@ namespace NKiwiAggr { SetFrame(newReferencePoint, newMaxValue, false); } -} +} diff --git a/library/cpp/histogram/adaptive/fixed_bin_histogram.h b/library/cpp/histogram/adaptive/fixed_bin_histogram.h index bd380bd94a..10106e6f54 100644 --- a/library/cpp/histogram/adaptive/fixed_bin_histogram.h +++ b/library/cpp/histogram/adaptive/fixed_bin_histogram.h @@ -15,7 +15,7 @@ namespace NKiwiAggr { static const size_t DEFAULT_INTERVALS = 100; typedef std::pair<double, double> TWeightedValue; // value, weight - THolder<TVector<TWeightedValue>> TrainingSet; + THolder<TVector<TWeightedValue>> TrainingSet; size_t TrainingSetSize; bool IsInitialized; @@ -40,7 +40,7 @@ namespace NKiwiAggr { TFixedBinHistogram(const THistogram& histo, size_t defaultIntervals = DEFAULT_INTERVALS, ui64 defaultId = 0, size_t trainingSetSize = DEFAULT_TRAINING_SET_SIZE); TFixedBinHistogram(IHistogram* histo, size_t defaultIntervals = DEFAULT_INTERVALS, ui64 defaultId = 0, size_t trainingSetSize = DEFAULT_TRAINING_SET_SIZE); - virtual ~TFixedBinHistogram() { + virtual ~TFixedBinHistogram() { } virtual void Clear(); @@ -88,4 +88,4 @@ namespace NKiwiAggr { } }; -} +} diff --git a/library/cpp/histogram/adaptive/histogram.h b/library/cpp/histogram/adaptive/histogram.h index 360fd9a693..a2ec2e46a4 100644 --- a/library/cpp/histogram/adaptive/histogram.h +++ b/library/cpp/histogram/adaptive/histogram.h @@ -8,7 +8,7 @@ namespace NKiwiAggr { class THistoRec; class IHistogram; - typedef TAtomicSharedPtr<IHistogram> IHistogramPtr; + typedef TAtomicSharedPtr<IHistogram> IHistogramPtr; class IHistogram { public: @@ -18,7 +18,7 @@ namespace NKiwiAggr { // TSomeHistogram(const THistogram& histo); // histo must be acceptable for TSomeHistogram, for example, only with HT_FIXED_BIN_HISTOGRAM for TFixedBinHistogram // TSomeHistogram(IHistogram* histo); // any kind of IHistogram - virtual ~IHistogram() { + virtual ~IHistogram() { } virtual void Clear() = 0; @@ -35,7 +35,7 @@ namespace NKiwiAggr { // factor should be greater then zero virtual void Multiply(double factor) = 0; - virtual void FromProto(const THistogram& histo) = 0; // throws exception in case of wrong histogram type of histo + virtual void FromProto(const THistogram& histo) = 0; // throws exception in case of wrong histogram type of histo virtual void ToProto(THistogram& histo) = 0; virtual void SetId(ui64 id) = 0; @@ -59,8 +59,8 @@ namespace NKiwiAggr { } // Histogram implementation is supposed to clear all precomputed values() if Add() is called after PrecomputePartialSums() - virtual void PrecomputePartialSums() { - } + virtual void PrecomputePartialSums() { + } }; -} +} diff --git a/library/cpp/histogram/adaptive/merger.h b/library/cpp/histogram/adaptive/merger.h index fc9a6b6a4f..6a40754610 100644 --- a/library/cpp/histogram/adaptive/merger.h +++ b/library/cpp/histogram/adaptive/merger.h @@ -6,7 +6,7 @@ namespace NKiwiAggr { class IMerger { private: bool IsMerged; - ui32 AutoMergeInterval; // Call Merge() after each AutoMergeInterval calls of Add(); zero means no autoMerge + ui32 AutoMergeInterval; // Call Merge() after each AutoMergeInterval calls of Add(); zero means no autoMerge ui32 NotMergedCount; public: @@ -17,7 +17,7 @@ namespace NKiwiAggr { { } - virtual ~IMerger() { + virtual ~IMerger() { } // returns true if something is added @@ -65,4 +65,4 @@ namespace NKiwiAggr { virtual void ResetImpl() = 0; }; -} +} diff --git a/library/cpp/histogram/adaptive/multi_histogram.h b/library/cpp/histogram/adaptive/multi_histogram.h index 41caac5ba6..187036cedd 100644 --- a/library/cpp/histogram/adaptive/multi_histogram.h +++ b/library/cpp/histogram/adaptive/multi_histogram.h @@ -31,7 +31,7 @@ namespace NKiwiAggr { FromProto(histograms); } - virtual ~TMultiHistogram() { + virtual ~TMultiHistogram() { } void Clear() { @@ -65,7 +65,7 @@ namespace NKiwiAggr { TVector<ui64> GetIds() const { TVector<ui64> result(0); for (THistogramsMap::const_iterator it = Histograms.begin(); it != Histograms.end(); ++it) { - result.push_back(it->first); + result.push_back(it->first); } return result; } @@ -81,7 +81,7 @@ namespace NKiwiAggr { double GetMaxHistoSum() const { double sum = 0.0; for (THistogramsMap::const_iterator it = Histograms.begin(); it != Histograms.end(); ++it) { - sum = std::max(sum, it->second->GetSum()); + sum = std::max(sum, it->second->GetSum()); } return sum; } @@ -138,6 +138,6 @@ namespace NKiwiAggr { // Good for parsing from THistograms protobuf typedef TMultiHistogram<TAutoHistogram> TAutoMultiHistogram; - typedef TAtomicSharedPtr<TAutoMultiHistogram> TAutoMultiHistogramPtr; + typedef TAtomicSharedPtr<TAutoMultiHistogram> TAutoMultiHistogramPtr; -} +} diff --git a/library/cpp/html/escape/escape.cpp b/library/cpp/html/escape/escape.cpp index 5b8ed60f04..4e38112ea1 100644 --- a/library/cpp/html/escape/escape.cpp +++ b/library/cpp/html/escape/escape.cpp @@ -4,63 +4,63 @@ #include <util/generic/strbuf.h> namespace NHtml { - namespace { - struct TReplace { - char Char; - bool ForText; - TStringBuf Entity; - }; + namespace { + struct TReplace { + char Char; + bool ForText; + TStringBuf Entity; + }; TReplace Escapable[] = { {'"', false, TStringBuf(""")}, {'&', true, TStringBuf("&")}, {'<', true, TStringBuf("<")}, {'>', true, TStringBuf(">")}, - }; + }; TString EscapeImpl(const TString& value, bool isText) { - auto ci = value.begin(); - // Looking for escapable characters. - for (; ci != value.end(); ++ci) { - for (size_t i = (isText ? 1 : 0); i < Y_ARRAY_SIZE(Escapable); ++i) { - if (*ci == Escapable[i].Char) { - goto escape; - } - } + auto ci = value.begin(); + // Looking for escapable characters. + for (; ci != value.end(); ++ci) { + for (size_t i = (isText ? 1 : 0); i < Y_ARRAY_SIZE(Escapable); ++i) { + if (*ci == Escapable[i].Char) { + goto escape; + } + } } - // There is no escapable characters, so return original value. - return value; + // There is no escapable characters, so return original value. + return value; - escape: - TString tmp = TString(value.begin(), ci); + escape: + TString tmp = TString(value.begin(), ci); - for (; ci != value.end(); ++ci) { - size_t i = (isText ? 1 : 0); + for (; ci != value.end(); ++ci) { + size_t i = (isText ? 1 : 0); - for (; i < Y_ARRAY_SIZE(Escapable); ++i) { - if (*ci == Escapable[i].Char) { - tmp += Escapable[i].Entity; - break; - } - } - - if (i == Y_ARRAY_SIZE(Escapable)) { - tmp += *ci; - } + for (; i < Y_ARRAY_SIZE(Escapable); ++i) { + if (*ci == Escapable[i].Char) { + tmp += Escapable[i].Entity; + break; + } + } + + if (i == Y_ARRAY_SIZE(Escapable)) { + tmp += *ci; + } } - - return tmp; + + return tmp; } } - TString EscapeAttributeValue(const TString& value) { - return EscapeImpl(value, false); - } + TString EscapeAttributeValue(const TString& value) { + return EscapeImpl(value, false); + } - TString EscapeText(const TString& value) { - return EscapeImpl(value, true); - } + TString EscapeText(const TString& value) { + return EscapeImpl(value, true); + } } diff --git a/library/cpp/html/escape/escape.h b/library/cpp/html/escape/escape.h index 1c45fc5193..e255ec7997 100644 --- a/library/cpp/html/escape/escape.h +++ b/library/cpp/html/escape/escape.h @@ -3,7 +3,7 @@ #include <util/generic/string.h> namespace NHtml { - TString EscapeAttributeValue(const TString& value); - TString EscapeText(const TString& value); + TString EscapeAttributeValue(const TString& value); + TString EscapeText(const TString& value); -} +} diff --git a/library/cpp/html/pcdata/pcdata.cpp b/library/cpp/html/pcdata/pcdata.cpp index 740c240fd2..1f8d2950de 100644 --- a/library/cpp/html/pcdata/pcdata.cpp +++ b/library/cpp/html/pcdata/pcdata.cpp @@ -1,54 +1,54 @@ -#include "pcdata.h" - +#include "pcdata.h" + #include <util/string/strspn.h> -static TCompactStrSpn sspn("\"<>&'"); - +static TCompactStrSpn sspn("\"<>&'"); + static void EncodeHtmlPcdataAppendInternal(const TStringBuf str, TString& strout, bool qAmp) { - const char* s = str.data(); - const char* e = s + str.length(); - - for (;;) { - const char* next = sspn.FindFirstOf(s, e); - - strout.AppendNoAlias(s, next - s); - s = next; - + const char* s = str.data(); + const char* e = s + str.length(); + + for (;;) { + const char* next = sspn.FindFirstOf(s, e); + + strout.AppendNoAlias(s, next - s); + s = next; + if (s == e) break; - - switch (*s) { - case '\"': + + switch (*s) { + case '\"': strout += TStringBuf("""); ++s; - break; - - case '<': + break; + + case '<': strout += TStringBuf("<"); ++s; - break; - - case '>': + break; + + case '>': strout += TStringBuf(">"); ++s; - break; - - case '\'': + break; + + case '\'': strout += TStringBuf("'"); ++s; - break; - - case '&': - if (qAmp) + break; + + case '&': + if (qAmp) strout += TStringBuf("&"); - else + else strout += TStringBuf("&"); ++s; - break; - } - } + break; + } + } } - + void EncodeHtmlPcdataAppend(const TStringBuf str, TString& strout) { EncodeHtmlPcdataAppendInternal(str, strout, true); } @@ -56,26 +56,26 @@ void EncodeHtmlPcdataAppend(const TStringBuf str, TString& strout) { TString EncodeHtmlPcdata(const TStringBuf str, bool qAmp) { TString strout; EncodeHtmlPcdataAppendInternal(str, strout, qAmp); - return strout; -} - + return strout; +} + TString DecodeHtmlPcdata(const TString& sz) { TString res; const char* codes[] = {""", "<", ">", "'", "'", "&", "'", nullptr}; - const char chars[] = {'\"', '<', '>', '\'', '\'', '&', '\''}; - for (size_t i = 0; i < sz.length(); ++i) { - char c = sz[i]; - if (c == '&') { - for (const char** p = codes; *p; ++p) { - size_t len = strlen(*p); - if (strncmp(sz.c_str() + i, *p, len) == 0) { - i += len - 1; - c = chars[p - codes]; - break; - } - } - } - res += c; - } - return res; -} + const char chars[] = {'\"', '<', '>', '\'', '\'', '&', '\''}; + for (size_t i = 0; i < sz.length(); ++i) { + char c = sz[i]; + if (c == '&') { + for (const char** p = codes; *p; ++p) { + size_t len = strlen(*p); + if (strncmp(sz.c_str() + i, *p, len) == 0) { + i += len - 1; + c = chars[p - codes]; + break; + } + } + } + res += c; + } + return res; +} diff --git a/library/cpp/html/pcdata/pcdata.h b/library/cpp/html/pcdata/pcdata.h index 7dd741f53d..f43c3d0c8d 100644 --- a/library/cpp/html/pcdata/pcdata.h +++ b/library/cpp/html/pcdata/pcdata.h @@ -1,10 +1,10 @@ -#pragma once - +#pragma once + #include <util/generic/fwd.h> - + /// Converts a text into HTML-code. Special characters of HTML («<», «>», ...) replaced with entities. TString EncodeHtmlPcdata(const TStringBuf str, bool qAmp = true); void EncodeHtmlPcdataAppend(const TStringBuf str, TString& strout); - + /// Reverse of EncodeHtmlPcdata() TString DecodeHtmlPcdata(const TString& sz); diff --git a/library/cpp/html/pcdata/pcdata_ut.cpp b/library/cpp/html/pcdata/pcdata_ut.cpp index 5833f8bc59..7418ecf11e 100644 --- a/library/cpp/html/pcdata/pcdata_ut.cpp +++ b/library/cpp/html/pcdata/pcdata_ut.cpp @@ -1,36 +1,36 @@ -#include "pcdata.h" - +#include "pcdata.h" + #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TPcdata) { Y_UNIT_TEST(TestStress) { - { - ui64 key = 0x000017C0B76C4E87ull; + { + ui64 key = 0x000017C0B76C4E87ull; TString res = EncodeHtmlPcdata(TStringBuf((const char*)&key, sizeof(key))); - } - - for (size_t i = 0; i < 1000; ++i) { + } + + for (size_t i = 0; i < 1000; ++i) { const TString s = NUnitTest::RandomString(i, i); - - UNIT_ASSERT_VALUES_EQUAL(DecodeHtmlPcdata(EncodeHtmlPcdata(s)), s); - } - } - + + UNIT_ASSERT_VALUES_EQUAL(DecodeHtmlPcdata(EncodeHtmlPcdata(s)), s); + } + } + Y_UNIT_TEST(Test1) { const TString tests[] = { - "qw&qw", - "&<", - ">&qw", - "\'&aaa"}; - + "qw&qw", + "&<", + ">&qw", + "\'&aaa"}; + for (auto s : tests) { - UNIT_ASSERT_VALUES_EQUAL(DecodeHtmlPcdata(EncodeHtmlPcdata(s)), s); - } - } - + UNIT_ASSERT_VALUES_EQUAL(DecodeHtmlPcdata(EncodeHtmlPcdata(s)), s); + } + } + Y_UNIT_TEST(Test2) { - UNIT_ASSERT_VALUES_EQUAL(EncodeHtmlPcdata("&qqq"), "&qqq"); - } + UNIT_ASSERT_VALUES_EQUAL(EncodeHtmlPcdata("&qqq"), "&qqq"); + } Y_UNIT_TEST(TestEncodeHtmlPcdataAppend) { TString s; @@ -41,8 +41,8 @@ Y_UNIT_TEST_SUITE(TPcdata) { } Y_UNIT_TEST(TestStrangeAmpParameter) { - UNIT_ASSERT_VALUES_EQUAL(EncodeHtmlPcdata("m&m's", true), "m&m's"); - UNIT_ASSERT_VALUES_EQUAL(EncodeHtmlPcdata("m&m's"), "m&m's"); //default + UNIT_ASSERT_VALUES_EQUAL(EncodeHtmlPcdata("m&m's", true), "m&m's"); + UNIT_ASSERT_VALUES_EQUAL(EncodeHtmlPcdata("m&m's"), "m&m's"); //default UNIT_ASSERT_VALUES_EQUAL(EncodeHtmlPcdata("m&m's", false), "m&m's"); } -} +} diff --git a/library/cpp/http/fetch/exthttpcodes.cpp b/library/cpp/http/fetch/exthttpcodes.cpp index acc05650c8..baa2a2a0fd 100644 --- a/library/cpp/http/fetch/exthttpcodes.cpp +++ b/library/cpp/http/fetch/exthttpcodes.cpp @@ -1,160 +1,160 @@ -#include "exthttpcodes.h" - +#include "exthttpcodes.h" + #include <cstring> -const ui16 CrazyServer = ShouldDelete | MarkSuspect; +const ui16 CrazyServer = ShouldDelete | MarkSuspect; -struct http_flag { - ui16 http; - ui16 flag; -}; +struct http_flag { + ui16 http; + ui16 flag; +}; static http_flag HTTP_FLAG[] = { - {HTTP_CONTINUE, MarkSuspect}, // 100 - {HTTP_SWITCHING_PROTOCOLS, CrazyServer}, // 101 - {HTTP_PROCESSING, CrazyServer}, // 102 + {HTTP_CONTINUE, MarkSuspect}, // 100 + {HTTP_SWITCHING_PROTOCOLS, CrazyServer}, // 101 + {HTTP_PROCESSING, CrazyServer}, // 102 - {HTTP_OK, ShouldReindex}, // 200 - {HTTP_CREATED, CrazyServer}, // 201 - {HTTP_ACCEPTED, ShouldDelete}, // 202 - {HTTP_NON_AUTHORITATIVE_INFORMATION, ShouldReindex}, // 203 - {HTTP_NO_CONTENT, ShouldDelete}, // 204 - {HTTP_RESET_CONTENT, ShouldDelete}, // 205 - {HTTP_PARTIAL_CONTENT, ShouldReindex}, // 206 - {HTTP_MULTI_STATUS, CrazyServer}, // 207 - {HTTP_ALREADY_REPORTED, CrazyServer}, // 208 - {HTTP_IM_USED, CrazyServer}, // 226 + {HTTP_OK, ShouldReindex}, // 200 + {HTTP_CREATED, CrazyServer}, // 201 + {HTTP_ACCEPTED, ShouldDelete}, // 202 + {HTTP_NON_AUTHORITATIVE_INFORMATION, ShouldReindex}, // 203 + {HTTP_NO_CONTENT, ShouldDelete}, // 204 + {HTTP_RESET_CONTENT, ShouldDelete}, // 205 + {HTTP_PARTIAL_CONTENT, ShouldReindex}, // 206 + {HTTP_MULTI_STATUS, CrazyServer}, // 207 + {HTTP_ALREADY_REPORTED, CrazyServer}, // 208 + {HTTP_IM_USED, CrazyServer}, // 226 - {HTTP_MULTIPLE_CHOICES, CheckLinks | ShouldDelete}, // 300 - {HTTP_MOVED_PERMANENTLY, CheckLocation | ShouldDelete | MoveRedir}, // 301 - {HTTP_FOUND, CheckLocation | ShouldDelete | MoveRedir}, // 302 - {HTTP_SEE_OTHER, CheckLocation | ShouldDelete | MoveRedir}, // 303 - {HTTP_NOT_MODIFIED, 0}, // 304 - {HTTP_USE_PROXY, ShouldDelete}, // 305 - {HTTP_TEMPORARY_REDIRECT, CheckLocation | ShouldDelete | MoveRedir}, // 307 - {HTTP_PERMANENT_REDIRECT, CheckLocation | ShouldDelete | MoveRedir}, // 308 + {HTTP_MULTIPLE_CHOICES, CheckLinks | ShouldDelete}, // 300 + {HTTP_MOVED_PERMANENTLY, CheckLocation | ShouldDelete | MoveRedir}, // 301 + {HTTP_FOUND, CheckLocation | ShouldDelete | MoveRedir}, // 302 + {HTTP_SEE_OTHER, CheckLocation | ShouldDelete | MoveRedir}, // 303 + {HTTP_NOT_MODIFIED, 0}, // 304 + {HTTP_USE_PROXY, ShouldDelete}, // 305 + {HTTP_TEMPORARY_REDIRECT, CheckLocation | ShouldDelete | MoveRedir}, // 307 + {HTTP_PERMANENT_REDIRECT, CheckLocation | ShouldDelete | MoveRedir}, // 308 - {HTTP_BAD_REQUEST, CrazyServer}, // 400 - {HTTP_UNAUTHORIZED, ShouldDelete}, // 401 - {HTTP_PAYMENT_REQUIRED, ShouldDelete}, // 402 - {HTTP_FORBIDDEN, ShouldDelete}, // 403 - {HTTP_NOT_FOUND, ShouldDelete}, // 404 - {HTTP_METHOD_NOT_ALLOWED, ShouldDelete}, // 405 - {HTTP_NOT_ACCEPTABLE, ShouldDelete}, // 406 - {HTTP_PROXY_AUTHENTICATION_REQUIRED, CrazyServer}, // 407 - {HTTP_REQUEST_TIME_OUT, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 408 - {HTTP_CONFLICT, MarkSuspect}, // 409 - {HTTP_GONE, ShouldDelete}, // 410 - {HTTP_LENGTH_REQUIRED, CrazyServer}, // 411 - {HTTP_PRECONDITION_FAILED, CrazyServer}, // 412 - {HTTP_REQUEST_ENTITY_TOO_LARGE, CrazyServer}, // 413 - {HTTP_REQUEST_URI_TOO_LARGE, ShouldDelete}, // 414 - {HTTP_UNSUPPORTED_MEDIA_TYPE, CrazyServer}, // 415 - {HTTP_REQUESTED_RANGE_NOT_SATISFIABLE, CrazyServer}, // 416 - {HTTP_EXPECTATION_FAILED, ShouldDelete}, // 417 - {HTTP_I_AM_A_TEAPOT, CrazyServer}, // 418 - {HTTP_AUTHENTICATION_TIMEOUT, ShouldDelete}, // 419 + {HTTP_BAD_REQUEST, CrazyServer}, // 400 + {HTTP_UNAUTHORIZED, ShouldDelete}, // 401 + {HTTP_PAYMENT_REQUIRED, ShouldDelete}, // 402 + {HTTP_FORBIDDEN, ShouldDelete}, // 403 + {HTTP_NOT_FOUND, ShouldDelete}, // 404 + {HTTP_METHOD_NOT_ALLOWED, ShouldDelete}, // 405 + {HTTP_NOT_ACCEPTABLE, ShouldDelete}, // 406 + {HTTP_PROXY_AUTHENTICATION_REQUIRED, CrazyServer}, // 407 + {HTTP_REQUEST_TIME_OUT, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 408 + {HTTP_CONFLICT, MarkSuspect}, // 409 + {HTTP_GONE, ShouldDelete}, // 410 + {HTTP_LENGTH_REQUIRED, CrazyServer}, // 411 + {HTTP_PRECONDITION_FAILED, CrazyServer}, // 412 + {HTTP_REQUEST_ENTITY_TOO_LARGE, CrazyServer}, // 413 + {HTTP_REQUEST_URI_TOO_LARGE, ShouldDelete}, // 414 + {HTTP_UNSUPPORTED_MEDIA_TYPE, CrazyServer}, // 415 + {HTTP_REQUESTED_RANGE_NOT_SATISFIABLE, CrazyServer}, // 416 + {HTTP_EXPECTATION_FAILED, ShouldDelete}, // 417 + {HTTP_I_AM_A_TEAPOT, CrazyServer}, // 418 + {HTTP_AUTHENTICATION_TIMEOUT, ShouldDelete}, // 419 - {HTTP_MISDIRECTED_REQUEST, CrazyServer}, // 421 - {HTTP_UNPROCESSABLE_ENTITY, CrazyServer}, // 422 - {HTTP_LOCKED, ShouldDelete}, // 423 - {HTTP_FAILED_DEPENDENCY, CrazyServer}, // 424 - {HTTP_UPGRADE_REQUIRED, ShouldDelete}, // 426 - {HTTP_PRECONDITION_REQUIRED, ShouldDelete}, // 428 - {HTTP_TOO_MANY_REQUESTS, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 429 - {HTTP_UNAVAILABLE_FOR_LEGAL_REASONS, ShouldDelete}, // 451 + {HTTP_MISDIRECTED_REQUEST, CrazyServer}, // 421 + {HTTP_UNPROCESSABLE_ENTITY, CrazyServer}, // 422 + {HTTP_LOCKED, ShouldDelete}, // 423 + {HTTP_FAILED_DEPENDENCY, CrazyServer}, // 424 + {HTTP_UPGRADE_REQUIRED, ShouldDelete}, // 426 + {HTTP_PRECONDITION_REQUIRED, ShouldDelete}, // 428 + {HTTP_TOO_MANY_REQUESTS, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 429 + {HTTP_UNAVAILABLE_FOR_LEGAL_REASONS, ShouldDelete}, // 451 - {HTTP_INTERNAL_SERVER_ERROR, MarkSuspect}, // 500 - {HTTP_NOT_IMPLEMENTED, ShouldDelete | ShouldDisconnect}, // 501 - {HTTP_BAD_GATEWAY, MarkSuspect}, // 502 - {HTTP_SERVICE_UNAVAILABLE, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 503 - {HTTP_GATEWAY_TIME_OUT, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 504 - {HTTP_HTTP_VERSION_NOT_SUPPORTED, CrazyServer | ShouldDisconnect}, // 505 + {HTTP_INTERNAL_SERVER_ERROR, MarkSuspect}, // 500 + {HTTP_NOT_IMPLEMENTED, ShouldDelete | ShouldDisconnect}, // 501 + {HTTP_BAD_GATEWAY, MarkSuspect}, // 502 + {HTTP_SERVICE_UNAVAILABLE, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 503 + {HTTP_GATEWAY_TIME_OUT, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 504 + {HTTP_HTTP_VERSION_NOT_SUPPORTED, CrazyServer | ShouldDisconnect}, // 505 - {HTTP_VARIANT_ALSO_NEGOTIATES, CrazyServer | ShouldDisconnect}, // 506 - {HTTP_INSUFFICIENT_STORAGE, CrazyServer | ShouldDisconnect}, // 507 - {HTTP_LOOP_DETECTED, CrazyServer | ShouldDisconnect}, // 508 - {HTTP_BANDWIDTH_LIMIT_EXCEEDED, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 509 - {HTTP_NOT_EXTENDED, ShouldDelete}, // 510 - {HTTP_NETWORK_AUTHENTICATION_REQUIRED, ShouldDelete}, // 511 + {HTTP_VARIANT_ALSO_NEGOTIATES, CrazyServer | ShouldDisconnect}, // 506 + {HTTP_INSUFFICIENT_STORAGE, CrazyServer | ShouldDisconnect}, // 507 + {HTTP_LOOP_DETECTED, CrazyServer | ShouldDisconnect}, // 508 + {HTTP_BANDWIDTH_LIMIT_EXCEEDED, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 509 + {HTTP_NOT_EXTENDED, ShouldDelete}, // 510 + {HTTP_NETWORK_AUTHENTICATION_REQUIRED, ShouldDelete}, // 511 - // custom - {HTTP_BAD_RESPONSE_HEADER, CrazyServer}, // 1000 - {HTTP_CONNECTION_LOST, ShouldRetry}, // 1001 - {HTTP_BODY_TOO_LARGE, ShouldDelete | CanBeFake}, // 1002 - {HTTP_ROBOTS_TXT_DISALLOW, ShouldDelete}, // 1003 - {HTTP_BAD_URL, ShouldDelete}, // 1004 - {HTTP_BAD_MIME, ShouldDelete}, // 1005 - {HTTP_DNS_FAILURE, ShouldDisconnect | MarkSuspect}, // 1006 - {HTTP_BAD_STATUS_CODE, CrazyServer}, // 1007 - {HTTP_BAD_HEADER_STRING, CrazyServer}, // 1008 - {HTTP_BAD_CHUNK, CrazyServer}, // 1009 - {HTTP_CONNECT_FAILED, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 1010 - {HTTP_FILTER_DISALLOW, ShouldDelete}, // 1011 - {HTTP_LOCAL_EIO, ShouldRetry}, // 1012 - {HTTP_BAD_CONTENT_LENGTH, ShouldDelete}, // 1013 - {HTTP_BAD_ENCODING, ShouldDelete}, // 1014 - {HTTP_LENGTH_UNKNOWN, ShouldDelete}, // 1015 - {HTTP_HEADER_EOF, ShouldRetry | CanBeFake}, // 1016 - {HTTP_MESSAGE_EOF, ShouldRetry | CanBeFake}, // 1017 - {HTTP_CHUNK_EOF, ShouldRetry | CanBeFake}, // 1018 - {HTTP_PAST_EOF, ShouldRetry | ShouldDelete | CanBeFake}, // 1019 - {HTTP_HEADER_TOO_LARGE, ShouldDelete}, // 1020 - {HTTP_URL_TOO_LARGE, ShouldDelete}, // 1021 - {HTTP_INTERRUPTED, 0}, // 1022 - {HTTP_CUSTOM_NOT_MODIFIED, 0}, // 1023 - {HTTP_BAD_CONTENT_ENCODING, ShouldDelete}, // 1024 - {HTTP_PROXY_UNKNOWN, 0}, // 1030 - {HTTP_PROXY_REQUEST_TIME_OUT, 0}, // 1031 - {HTTP_PROXY_INTERNAL_ERROR, 0}, // 1032 - {HTTP_PROXY_CONNECT_FAILED, 0}, // 1033 - {HTTP_PROXY_CONNECTION_LOST, 0}, // 1034 - {HTTP_PROXY_NO_PROXY, 0}, // 1035 - {HTTP_PROXY_ERROR, 0}, // 1036 - {HTTP_SSL_ERROR, 0}, // 1037 - {HTTP_CACHED_COPY_NOT_FOUND, 0}, // 1038 - {HTTP_TIMEDOUT_WHILE_BYTES_RECEIVING, ShouldRetry}, // 1039 - {HTTP_FETCHER_BAD_RESPONSE, 0}, // 1040 - {HTTP_FETCHER_MB_ERROR, 0}, // 1041 - {HTTP_SSL_CERT_ERROR, 0}, // 1042 + // custom + {HTTP_BAD_RESPONSE_HEADER, CrazyServer}, // 1000 + {HTTP_CONNECTION_LOST, ShouldRetry}, // 1001 + {HTTP_BODY_TOO_LARGE, ShouldDelete | CanBeFake}, // 1002 + {HTTP_ROBOTS_TXT_DISALLOW, ShouldDelete}, // 1003 + {HTTP_BAD_URL, ShouldDelete}, // 1004 + {HTTP_BAD_MIME, ShouldDelete}, // 1005 + {HTTP_DNS_FAILURE, ShouldDisconnect | MarkSuspect}, // 1006 + {HTTP_BAD_STATUS_CODE, CrazyServer}, // 1007 + {HTTP_BAD_HEADER_STRING, CrazyServer}, // 1008 + {HTTP_BAD_CHUNK, CrazyServer}, // 1009 + {HTTP_CONNECT_FAILED, ShouldDisconnect | ShouldRetry | MarkSuspect}, // 1010 + {HTTP_FILTER_DISALLOW, ShouldDelete}, // 1011 + {HTTP_LOCAL_EIO, ShouldRetry}, // 1012 + {HTTP_BAD_CONTENT_LENGTH, ShouldDelete}, // 1013 + {HTTP_BAD_ENCODING, ShouldDelete}, // 1014 + {HTTP_LENGTH_UNKNOWN, ShouldDelete}, // 1015 + {HTTP_HEADER_EOF, ShouldRetry | CanBeFake}, // 1016 + {HTTP_MESSAGE_EOF, ShouldRetry | CanBeFake}, // 1017 + {HTTP_CHUNK_EOF, ShouldRetry | CanBeFake}, // 1018 + {HTTP_PAST_EOF, ShouldRetry | ShouldDelete | CanBeFake}, // 1019 + {HTTP_HEADER_TOO_LARGE, ShouldDelete}, // 1020 + {HTTP_URL_TOO_LARGE, ShouldDelete}, // 1021 + {HTTP_INTERRUPTED, 0}, // 1022 + {HTTP_CUSTOM_NOT_MODIFIED, 0}, // 1023 + {HTTP_BAD_CONTENT_ENCODING, ShouldDelete}, // 1024 + {HTTP_PROXY_UNKNOWN, 0}, // 1030 + {HTTP_PROXY_REQUEST_TIME_OUT, 0}, // 1031 + {HTTP_PROXY_INTERNAL_ERROR, 0}, // 1032 + {HTTP_PROXY_CONNECT_FAILED, 0}, // 1033 + {HTTP_PROXY_CONNECTION_LOST, 0}, // 1034 + {HTTP_PROXY_NO_PROXY, 0}, // 1035 + {HTTP_PROXY_ERROR, 0}, // 1036 + {HTTP_SSL_ERROR, 0}, // 1037 + {HTTP_CACHED_COPY_NOT_FOUND, 0}, // 1038 + {HTTP_TIMEDOUT_WHILE_BYTES_RECEIVING, ShouldRetry}, // 1039 + {HTTP_FETCHER_BAD_RESPONSE, 0}, // 1040 + {HTTP_FETCHER_MB_ERROR, 0}, // 1041 + {HTTP_SSL_CERT_ERROR, 0}, // 1042 - // Custom (replace HTTP 200/304) - {EXT_HTTP_MIRRMOVE, 0}, // 2000 - {EXT_HTTP_MANUAL_DELETE, ShouldDelete}, // 2001 - {EXT_HTTP_NOTUSED2, ShouldDelete}, // 2002 - {EXT_HTTP_NOTUSED3, ShouldDelete}, // 2003 - {EXT_HTTP_REFRESH, ShouldDelete | CheckLinks | MoveRedir}, // 2004 - {EXT_HTTP_NOINDEX, ShouldDelete | CheckLinks}, // 2005 - {EXT_HTTP_BADCODES, ShouldDelete}, // 2006 - {EXT_HTTP_SITESTAT, ShouldDelete}, // 2007 - {EXT_HTTP_IOERROR, ShouldDelete}, // 2008 - {EXT_HTTP_BASEERROR, ShouldDelete}, // 2009 - {EXT_HTTP_PARSERROR, ShouldDelete | CanBeFake}, // 2010 - {EXT_HTTP_BAD_CHARSET, ShouldDelete | CheckLinks}, // 2011 - {EXT_HTTP_BAD_LANGUAGE, ShouldDelete | CheckLinks}, // 2012 - {EXT_HTTP_NUMERERROR, ShouldDelete}, // 2013 - {EXT_HTTP_EMPTYDOC, ShouldDelete | CheckLinks}, // 2014 - {EXT_HTTP_HUGEDOC, ShouldDelete}, // 2015 - {EXT_HTTP_LINKGARBAGE, ShouldDelete}, // 2016 - {EXT_HTTP_PARSERFAIL, ShouldDelete}, // 2019 - {EXT_HTTP_GZIPERROR, ShouldDelete}, // 2020 - {EXT_HTTP_MANUAL_DELETE_URL, ShouldDelete}, // 2022 - {EXT_HTTP_CUSTOM_PARTIAL_CONTENT, ShouldReindex}, // 2023 - {EXT_HTTP_EMPTY_RESPONSE, ShouldDelete}, // 2024 - {EXT_HTTP_REL_CANONICAL, ShouldDelete | CheckLinks | MoveRedir}, // 2025 - {0, 0}}; + // Custom (replace HTTP 200/304) + {EXT_HTTP_MIRRMOVE, 0}, // 2000 + {EXT_HTTP_MANUAL_DELETE, ShouldDelete}, // 2001 + {EXT_HTTP_NOTUSED2, ShouldDelete}, // 2002 + {EXT_HTTP_NOTUSED3, ShouldDelete}, // 2003 + {EXT_HTTP_REFRESH, ShouldDelete | CheckLinks | MoveRedir}, // 2004 + {EXT_HTTP_NOINDEX, ShouldDelete | CheckLinks}, // 2005 + {EXT_HTTP_BADCODES, ShouldDelete}, // 2006 + {EXT_HTTP_SITESTAT, ShouldDelete}, // 2007 + {EXT_HTTP_IOERROR, ShouldDelete}, // 2008 + {EXT_HTTP_BASEERROR, ShouldDelete}, // 2009 + {EXT_HTTP_PARSERROR, ShouldDelete | CanBeFake}, // 2010 + {EXT_HTTP_BAD_CHARSET, ShouldDelete | CheckLinks}, // 2011 + {EXT_HTTP_BAD_LANGUAGE, ShouldDelete | CheckLinks}, // 2012 + {EXT_HTTP_NUMERERROR, ShouldDelete}, // 2013 + {EXT_HTTP_EMPTYDOC, ShouldDelete | CheckLinks}, // 2014 + {EXT_HTTP_HUGEDOC, ShouldDelete}, // 2015 + {EXT_HTTP_LINKGARBAGE, ShouldDelete}, // 2016 + {EXT_HTTP_PARSERFAIL, ShouldDelete}, // 2019 + {EXT_HTTP_GZIPERROR, ShouldDelete}, // 2020 + {EXT_HTTP_MANUAL_DELETE_URL, ShouldDelete}, // 2022 + {EXT_HTTP_CUSTOM_PARTIAL_CONTENT, ShouldReindex}, // 2023 + {EXT_HTTP_EMPTY_RESPONSE, ShouldDelete}, // 2024 + {EXT_HTTP_REL_CANONICAL, ShouldDelete | CheckLinks | MoveRedir}, // 2025 + {0, 0}}; -static ui16* prepare_flags(http_flag* arg) { +static ui16* prepare_flags(http_flag* arg) { static ui16 flags[EXT_HTTP_CODE_MAX]; - http_flag* ptr; + http_flag* ptr; size_t i; // устанавливаем значение по умолчанию для кодов не перечисленных в таблице выше - for (i = 0; i < EXT_HTTP_CODE_MAX; ++i) + for (i = 0; i < EXT_HTTP_CODE_MAX; ++i) flags[i] = CrazyServer; // устанавливаем флаги для перечисленных кодов - for (ptr = arg; ptr->http; ++ptr) - flags[ptr->http & (EXT_HTTP_CODE_MAX - 1)] = ptr->flag; + for (ptr = arg; ptr->http; ++ptr) + flags[ptr->http & (EXT_HTTP_CODE_MAX - 1)] = ptr->flag; // для стандартных кодов ошибок берем флаги из первого кода каждой группы и проставляем их // всем кодам не перечисленным в таблице выше @@ -164,103 +164,103 @@ static ui16* prepare_flags(http_flag* arg) { // предыдущий цикл затер некоторые флаги перечисленные в таблице выше // восстанавливаем их - for (ptr = arg; ptr->http; ++ptr) - flags[ptr->http & (EXT_HTTP_CODE_MAX - 1)] = ptr->flag; + for (ptr = arg; ptr->http; ++ptr) + flags[ptr->http & (EXT_HTTP_CODE_MAX - 1)] = ptr->flag; return flags; } -ui16* http2status = prepare_flags(HTTP_FLAG); +ui16* http2status = prepare_flags(HTTP_FLAG); TStringBuf ExtHttpCodeStr(int code) noexcept { if (code < HTTP_CODE_MAX) { return HttpCodeStr(code); } switch (code) { - case HTTP_BAD_RESPONSE_HEADER: + case HTTP_BAD_RESPONSE_HEADER: return TStringBuf("Bad response header"); - case HTTP_CONNECTION_LOST: + case HTTP_CONNECTION_LOST: return TStringBuf("Connection lost"); - case HTTP_BODY_TOO_LARGE: + case HTTP_BODY_TOO_LARGE: return TStringBuf("Body too large"); - case HTTP_ROBOTS_TXT_DISALLOW: + case HTTP_ROBOTS_TXT_DISALLOW: return TStringBuf("robots.txt disallow"); - case HTTP_BAD_URL: + case HTTP_BAD_URL: return TStringBuf("Bad url"); - case HTTP_BAD_MIME: + case HTTP_BAD_MIME: return TStringBuf("Bad mime type"); - case HTTP_DNS_FAILURE: + case HTTP_DNS_FAILURE: return TStringBuf("Dns failure"); - case HTTP_BAD_STATUS_CODE: + case HTTP_BAD_STATUS_CODE: return TStringBuf("Bad status code"); - case HTTP_BAD_HEADER_STRING: + case HTTP_BAD_HEADER_STRING: return TStringBuf("Bad header string"); - case HTTP_BAD_CHUNK: + case HTTP_BAD_CHUNK: return TStringBuf("Bad chunk"); - case HTTP_CONNECT_FAILED: + case HTTP_CONNECT_FAILED: return TStringBuf("Connect failed"); - case HTTP_FILTER_DISALLOW: + case HTTP_FILTER_DISALLOW: return TStringBuf("Filter disallow"); - case HTTP_LOCAL_EIO: + case HTTP_LOCAL_EIO: return TStringBuf("Local eio"); - case HTTP_BAD_CONTENT_LENGTH: + case HTTP_BAD_CONTENT_LENGTH: return TStringBuf("Bad content length"); - case HTTP_BAD_ENCODING: + case HTTP_BAD_ENCODING: return TStringBuf("Bad encoding"); - case HTTP_LENGTH_UNKNOWN: + case HTTP_LENGTH_UNKNOWN: return TStringBuf("Length unknown"); - case HTTP_HEADER_EOF: + case HTTP_HEADER_EOF: return TStringBuf("Header EOF"); - case HTTP_MESSAGE_EOF: + case HTTP_MESSAGE_EOF: return TStringBuf("Message EOF"); - case HTTP_CHUNK_EOF: + case HTTP_CHUNK_EOF: return TStringBuf("Chunk EOF"); - case HTTP_PAST_EOF: + case HTTP_PAST_EOF: return TStringBuf("Past EOF"); - case HTTP_HEADER_TOO_LARGE: + case HTTP_HEADER_TOO_LARGE: return TStringBuf("Header is too large"); - case HTTP_URL_TOO_LARGE: + case HTTP_URL_TOO_LARGE: return TStringBuf("Url is too large"); - case HTTP_INTERRUPTED: + case HTTP_INTERRUPTED: return TStringBuf("Interrupted"); - case HTTP_CUSTOM_NOT_MODIFIED: + case HTTP_CUSTOM_NOT_MODIFIED: return TStringBuf("Signature detector thinks that doc is not modified"); - case HTTP_BAD_CONTENT_ENCODING: + case HTTP_BAD_CONTENT_ENCODING: return TStringBuf("Bad content encoding"); - case HTTP_NO_RESOURCES: + case HTTP_NO_RESOURCES: return TStringBuf("No resources"); - case HTTP_FETCHER_SHUTDOWN: + case HTTP_FETCHER_SHUTDOWN: return TStringBuf("Fetcher shutdown"); - case HTTP_CHUNK_TOO_LARGE: + case HTTP_CHUNK_TOO_LARGE: return TStringBuf("Chunk size is too big"); - case HTTP_SERVER_BUSY: + case HTTP_SERVER_BUSY: return TStringBuf("Server is busy"); - case HTTP_SERVICE_UNKNOWN: + case HTTP_SERVICE_UNKNOWN: return TStringBuf("Service is unknown"); - case HTTP_PROXY_UNKNOWN: + case HTTP_PROXY_UNKNOWN: return TStringBuf("Zora: unknown error"); - case HTTP_PROXY_REQUEST_TIME_OUT: + case HTTP_PROXY_REQUEST_TIME_OUT: return TStringBuf("Zora: request time out"); - case HTTP_PROXY_INTERNAL_ERROR: + case HTTP_PROXY_INTERNAL_ERROR: return TStringBuf("Zora: internal server error"); - case HTTP_PROXY_CONNECT_FAILED: + case HTTP_PROXY_CONNECT_FAILED: return TStringBuf("Spider proxy connect failed"); - case HTTP_PROXY_CONNECTION_LOST: + case HTTP_PROXY_CONNECTION_LOST: return TStringBuf("Spider proxy connection lost"); - case HTTP_PROXY_NO_PROXY: + case HTTP_PROXY_NO_PROXY: return TStringBuf("Spider proxy no proxy alive in region"); - case HTTP_PROXY_ERROR: + case HTTP_PROXY_ERROR: return TStringBuf("Spider proxy returned custom error"); - case HTTP_SSL_ERROR: + case HTTP_SSL_ERROR: return TStringBuf("Ssl library returned error"); - case HTTP_CACHED_COPY_NOT_FOUND: + case HTTP_CACHED_COPY_NOT_FOUND: return TStringBuf("Cached copy for the url is not available"); - case HTTP_TIMEDOUT_WHILE_BYTES_RECEIVING: + case HTTP_TIMEDOUT_WHILE_BYTES_RECEIVING: return TStringBuf("Timed out while bytes receiving"); - // TODO: messages for >2000 codes + // TODO: messages for >2000 codes - default: + default: return TStringBuf("Unknown HTTP code"); } } diff --git a/library/cpp/http/fetch/exthttpcodes.h b/library/cpp/http/fetch/exthttpcodes.h index 6b525052cd..537e4ac6df 100644 --- a/library/cpp/http/fetch/exthttpcodes.h +++ b/library/cpp/http/fetch/exthttpcodes.h @@ -5,110 +5,110 @@ enum ExtHttpCodes { // Custom - HTTP_EXTENDED = 1000, - HTTP_BAD_RESPONSE_HEADER = 1000, - HTTP_CONNECTION_LOST = 1001, - HTTP_BODY_TOO_LARGE = 1002, - HTTP_ROBOTS_TXT_DISALLOW = 1003, - HTTP_BAD_URL = 1004, - HTTP_BAD_MIME = 1005, - HTTP_DNS_FAILURE = 1006, - HTTP_BAD_STATUS_CODE = 1007, - HTTP_BAD_HEADER_STRING = 1008, - HTTP_BAD_CHUNK = 1009, - HTTP_CONNECT_FAILED = 1010, - HTTP_FILTER_DISALLOW = 1011, - HTTP_LOCAL_EIO = 1012, - HTTP_BAD_CONTENT_LENGTH = 1013, - HTTP_BAD_ENCODING = 1014, - HTTP_LENGTH_UNKNOWN = 1015, - HTTP_HEADER_EOF = 1016, - HTTP_MESSAGE_EOF = 1017, - HTTP_CHUNK_EOF = 1018, - HTTP_PAST_EOF = 1019, - HTTP_HEADER_TOO_LARGE = 1020, - HTTP_URL_TOO_LARGE = 1021, - HTTP_INTERRUPTED = 1022, - HTTP_CUSTOM_NOT_MODIFIED = 1023, - HTTP_BAD_CONTENT_ENCODING = 1024, - HTTP_NO_RESOURCES = 1025, - HTTP_FETCHER_SHUTDOWN = 1026, - HTTP_CHUNK_TOO_LARGE = 1027, - HTTP_SERVER_BUSY = 1028, - HTTP_SERVICE_UNKNOWN = 1029, - HTTP_PROXY_UNKNOWN = 1030, - HTTP_PROXY_REQUEST_TIME_OUT = 1031, - HTTP_PROXY_INTERNAL_ERROR = 1032, - HTTP_PROXY_CONNECT_FAILED = 1033, - HTTP_PROXY_CONNECTION_LOST = 1034, - HTTP_PROXY_NO_PROXY = 1035, - HTTP_PROXY_ERROR = 1036, - HTTP_SSL_ERROR = 1037, - HTTP_CACHED_COPY_NOT_FOUND = 1038, + HTTP_EXTENDED = 1000, + HTTP_BAD_RESPONSE_HEADER = 1000, + HTTP_CONNECTION_LOST = 1001, + HTTP_BODY_TOO_LARGE = 1002, + HTTP_ROBOTS_TXT_DISALLOW = 1003, + HTTP_BAD_URL = 1004, + HTTP_BAD_MIME = 1005, + HTTP_DNS_FAILURE = 1006, + HTTP_BAD_STATUS_CODE = 1007, + HTTP_BAD_HEADER_STRING = 1008, + HTTP_BAD_CHUNK = 1009, + HTTP_CONNECT_FAILED = 1010, + HTTP_FILTER_DISALLOW = 1011, + HTTP_LOCAL_EIO = 1012, + HTTP_BAD_CONTENT_LENGTH = 1013, + HTTP_BAD_ENCODING = 1014, + HTTP_LENGTH_UNKNOWN = 1015, + HTTP_HEADER_EOF = 1016, + HTTP_MESSAGE_EOF = 1017, + HTTP_CHUNK_EOF = 1018, + HTTP_PAST_EOF = 1019, + HTTP_HEADER_TOO_LARGE = 1020, + HTTP_URL_TOO_LARGE = 1021, + HTTP_INTERRUPTED = 1022, + HTTP_CUSTOM_NOT_MODIFIED = 1023, + HTTP_BAD_CONTENT_ENCODING = 1024, + HTTP_NO_RESOURCES = 1025, + HTTP_FETCHER_SHUTDOWN = 1026, + HTTP_CHUNK_TOO_LARGE = 1027, + HTTP_SERVER_BUSY = 1028, + HTTP_SERVICE_UNKNOWN = 1029, + HTTP_PROXY_UNKNOWN = 1030, + HTTP_PROXY_REQUEST_TIME_OUT = 1031, + HTTP_PROXY_INTERNAL_ERROR = 1032, + HTTP_PROXY_CONNECT_FAILED = 1033, + HTTP_PROXY_CONNECTION_LOST = 1034, + HTTP_PROXY_NO_PROXY = 1035, + HTTP_PROXY_ERROR = 1036, + HTTP_SSL_ERROR = 1037, + HTTP_CACHED_COPY_NOT_FOUND = 1038, HTTP_TIMEDOUT_WHILE_BYTES_RECEIVING = 1039, - HTTP_FETCHER_BAD_RESPONSE = 1040, - HTTP_FETCHER_MB_ERROR = 1041, - HTTP_SSL_CERT_ERROR = 1042, - HTTP_PROXY_REQUEST_CANCELED = 1051, + HTTP_FETCHER_BAD_RESPONSE = 1040, + HTTP_FETCHER_MB_ERROR = 1041, + HTTP_SSL_CERT_ERROR = 1042, + HTTP_PROXY_REQUEST_CANCELED = 1051, // Custom (replace HTTP 200/304) - EXT_HTTP_EXT_SUCCESS_BEGIN = 2000, // to check if code variable is in success interval - EXT_HTTP_MIRRMOVE = 2000, - EXT_HTTP_MANUAL_DELETE = 2001, - EXT_HTTP_NOTUSED2 = 2002, - EXT_HTTP_NOTUSED3 = 2003, - EXT_HTTP_REFRESH = 2004, - EXT_HTTP_NOINDEX = 2005, - EXT_HTTP_BADCODES = 2006, - EXT_HTTP_SITESTAT = 2007, - EXT_HTTP_IOERROR = 2008, - EXT_HTTP_BASEERROR = 2009, - EXT_HTTP_PARSERROR = 2010, - EXT_HTTP_BAD_CHARSET = 2011, - EXT_HTTP_BAD_LANGUAGE = 2012, - EXT_HTTP_NUMERERROR = 2013, - EXT_HTTP_EMPTYDOC = 2014, - EXT_HTTP_HUGEDOC = 2015, - EXT_HTTP_LINKGARBAGE = 2016, - EXT_HTTP_EXDUPLICATE = 2017, - EXT_HTTP_FILTERED = 2018, - EXT_HTTP_PARSERFAIL = 2019, // parser crashed (in this case image spider will redownload such document) - EXT_HTTP_GZIPERROR = 2020, - EXT_HTTP_CLEANPARAM = 2021, - EXT_HTTP_MANUAL_DELETE_URL = 2022, - EXT_HTTP_CUSTOM_PARTIAL_CONTENT = 2023, - EXT_HTTP_EMPTY_RESPONSE = 2024, - EXT_HTTP_REL_CANONICAL = 2025, + EXT_HTTP_EXT_SUCCESS_BEGIN = 2000, // to check if code variable is in success interval + EXT_HTTP_MIRRMOVE = 2000, + EXT_HTTP_MANUAL_DELETE = 2001, + EXT_HTTP_NOTUSED2 = 2002, + EXT_HTTP_NOTUSED3 = 2003, + EXT_HTTP_REFRESH = 2004, + EXT_HTTP_NOINDEX = 2005, + EXT_HTTP_BADCODES = 2006, + EXT_HTTP_SITESTAT = 2007, + EXT_HTTP_IOERROR = 2008, + EXT_HTTP_BASEERROR = 2009, + EXT_HTTP_PARSERROR = 2010, + EXT_HTTP_BAD_CHARSET = 2011, + EXT_HTTP_BAD_LANGUAGE = 2012, + EXT_HTTP_NUMERERROR = 2013, + EXT_HTTP_EMPTYDOC = 2014, + EXT_HTTP_HUGEDOC = 2015, + EXT_HTTP_LINKGARBAGE = 2016, + EXT_HTTP_EXDUPLICATE = 2017, + EXT_HTTP_FILTERED = 2018, + EXT_HTTP_PARSERFAIL = 2019, // parser crashed (in this case image spider will redownload such document) + EXT_HTTP_GZIPERROR = 2020, + EXT_HTTP_CLEANPARAM = 2021, + EXT_HTTP_MANUAL_DELETE_URL = 2022, + EXT_HTTP_CUSTOM_PARTIAL_CONTENT = 2023, + EXT_HTTP_EMPTY_RESPONSE = 2024, + EXT_HTTP_REL_CANONICAL = 2025, - EXT_HTTP_EXT_SUCCESS_END = 3000, // to check if code variable is in success interval - EXT_HTTP_HOSTFILTER = 3001, - EXT_HTTP_URLFILTER = 3002, - EXT_HTTP_SUFFIXFILTER = 3003, - EXT_HTTP_DOMAINFILTER = 3004, - EXT_HTTP_EXTDOMAINFILTER = 3005, - EXT_HTTP_PORTFILTER = 3006, - EXT_HTTP_MIRROR = 3007, - EXT_HTTP_DEEPDIR = 3008, - EXT_HTTP_DUPDIRS = 3009, - EXT_HTTP_REGEXP = 3010, - EXT_HTTP_OLDDELETED = 3012, - EXT_HTTP_PENALTY = 3013, - EXT_HTTP_POLICY = 3015, - EXT_HTTP_TOOOLD = 3016, - EXT_HTTP_GARBAGE = 3017, - EXT_HTTP_FOREIGN = 3018, - EXT_HTTP_EXT_REGEXP = 3019, - EXT_HTTP_HOPS = 3020, - EXT_HTTP_SELRANK = 3021, - EXT_HTTP_NOLINKS = 3022, - EXT_HTTP_WRONGMULTILANG = 3023, - EXT_HTTP_SOFTMIRRORS = 3024, - EXT_HTTP_BIGLEVEL = 3025, + EXT_HTTP_EXT_SUCCESS_END = 3000, // to check if code variable is in success interval + EXT_HTTP_HOSTFILTER = 3001, + EXT_HTTP_URLFILTER = 3002, + EXT_HTTP_SUFFIXFILTER = 3003, + EXT_HTTP_DOMAINFILTER = 3004, + EXT_HTTP_EXTDOMAINFILTER = 3005, + EXT_HTTP_PORTFILTER = 3006, + EXT_HTTP_MIRROR = 3007, + EXT_HTTP_DEEPDIR = 3008, + EXT_HTTP_DUPDIRS = 3009, + EXT_HTTP_REGEXP = 3010, + EXT_HTTP_OLDDELETED = 3012, + EXT_HTTP_PENALTY = 3013, + EXT_HTTP_POLICY = 3015, + EXT_HTTP_TOOOLD = 3016, + EXT_HTTP_GARBAGE = 3017, + EXT_HTTP_FOREIGN = 3018, + EXT_HTTP_EXT_REGEXP = 3019, + EXT_HTTP_HOPS = 3020, + EXT_HTTP_SELRANK = 3021, + EXT_HTTP_NOLINKS = 3022, + EXT_HTTP_WRONGMULTILANG = 3023, + EXT_HTTP_SOFTMIRRORS = 3024, + EXT_HTTP_BIGLEVEL = 3025, // fast robot codes - EXT_HTTP_FASTHOPS = 4000, - EXT_HTTP_NODOC = 4001, + EXT_HTTP_FASTHOPS = 4000, + EXT_HTTP_NODOC = 4001, EXT_HTTP_MAX }; @@ -116,26 +116,26 @@ enum ExtHttpCodes { enum HttpFlags { // connection ShouldDisconnect = 1, - ShouldRetry = 2, + ShouldRetry = 2, // UNUSED 4 // indexer - ShouldReindex = 8, - ShouldDelete = 16, - CheckLocation = 32, - CheckLinks = 64, - MarkSuspect = 128, + ShouldReindex = 8, + ShouldDelete = 16, + CheckLocation = 32, + CheckLinks = 64, + MarkSuspect = 128, // UNUSED 256 // UNUSED 512 - MoveRedir = 1024, - CanBeFake = 2048, + MoveRedir = 1024, + CanBeFake = 2048, }; const size_t EXT_HTTP_CODE_MAX = 1 << 12; static inline int Http2Status(int code) { - extern ui16* http2status; - return http2status[code & (EXT_HTTP_CODE_MAX - 1)]; + extern ui16* http2status; + return http2status[code & (EXT_HTTP_CODE_MAX - 1)]; } TStringBuf ExtHttpCodeStr(int code) noexcept; diff --git a/library/cpp/http/fetch/http_digest.cpp b/library/cpp/http/fetch/http_digest.cpp index 1eaa02b7f2..79c5a25319 100644 --- a/library/cpp/http/fetch/http_digest.cpp +++ b/library/cpp/http/fetch/http_digest.cpp @@ -1,5 +1,5 @@ -#include "http_digest.h" - +#include "http_digest.h" + #include <library/cpp/digest/md5/md5.h> #include <util/stream/output.h> #include <util/stream/str.h> @@ -9,76 +9,76 @@ static const char* WWW_PREFIX = "Authorization: Digest "; /************************************************************/ -httpDigestHandler::httpDigestHandler() - : User_(nullptr) - , Password_(nullptr) - , Nonce_(nullptr) - , NonceCount_(0) - , HeaderInstruction_(nullptr) +httpDigestHandler::httpDigestHandler() + : User_(nullptr) + , Password_(nullptr) + , Nonce_(nullptr) + , NonceCount_(0) + , HeaderInstruction_(nullptr) { } /************************************************************/ -httpDigestHandler::~httpDigestHandler() { +httpDigestHandler::~httpDigestHandler() { clear(); } /************************************************************/ -void httpDigestHandler::clear() { - free(Nonce_); - free(HeaderInstruction_); +void httpDigestHandler::clear() { + free(Nonce_); + free(HeaderInstruction_); User_ = Password_ = nullptr; Nonce_ = HeaderInstruction_ = nullptr; - NonceCount_ = 0; + NonceCount_ = 0; } /************************************************************/ -void httpDigestHandler::setAuthorization(const char* user, const char* password) { +void httpDigestHandler::setAuthorization(const char* user, const char* password) { clear(); - if (user && password) { - User_ = user; - Password_ = password; + if (user && password) { + User_ = user; + Password_ = password; } } /************************************************************/ -const char* httpDigestHandler::getHeaderInstruction() const { - return HeaderInstruction_; +const char* httpDigestHandler::getHeaderInstruction() const { + return HeaderInstruction_; } /************************************************************/ -void httpDigestHandler::generateCNonce(char* outCNonce) { +void httpDigestHandler::generateCNonce(char* outCNonce) { if (!*outCNonce) sprintf(outCNonce, "%ld", (long)time(nullptr)); } /************************************************************/ -inline void addMD5(MD5& ctx, const char* value) { +inline void addMD5(MD5& ctx, const char* value) { ctx.Update((const unsigned char*)(value), strlen(value)); } -inline void addMD5(MD5& ctx, const char* value, int len) { +inline void addMD5(MD5& ctx, const char* value, int len) { ctx.Update((const unsigned char*)(value), len); } -inline void addMD5Sep(MD5& ctx) { +inline void addMD5Sep(MD5& ctx) { addMD5(ctx, ":", 1); } /************************************************************/ /* calculate H(A1) as per spec */ -void httpDigestHandler::digestCalcHA1(const THttpAuthHeader& hd, - char* outSessionKey, - char* outCNonce) { +void httpDigestHandler::digestCalcHA1(const THttpAuthHeader& hd, + char* outSessionKey, + char* outCNonce) { MD5 ctx; ctx.Init(); - addMD5(ctx, User_); - addMD5Sep(ctx); - addMD5(ctx, hd.realm); - addMD5Sep(ctx); - addMD5(ctx, Password_); + addMD5(ctx, User_); + addMD5Sep(ctx); + addMD5(ctx, hd.realm); + addMD5Sep(ctx); + addMD5(ctx, Password_); - if (hd.algorithm == 1) { //MD5-sess + if (hd.algorithm == 1) { //MD5-sess unsigned char digest[16]; ctx.Final(digest); @@ -86,91 +86,91 @@ void httpDigestHandler::digestCalcHA1(const THttpAuthHeader& hd, ctx.Init(); ctx.Update(digest, 16); - addMD5Sep(ctx); - addMD5(ctx, hd.nonce); - addMD5Sep(ctx); - addMD5(ctx, outCNonce); + addMD5Sep(ctx); + addMD5(ctx, hd.nonce); + addMD5Sep(ctx); + addMD5(ctx, outCNonce); ctx.End(outSessionKey); } - ctx.End(outSessionKey); + ctx.End(outSessionKey); }; /************************************************************/ /* calculate request-digest/response-digest as per HTTP Digest spec */ -void httpDigestHandler::digestCalcResponse(const THttpAuthHeader& hd, - const char* path, - const char* method, - const char* nonceCount, - char* outResponse, - char* outCNonce) { +void httpDigestHandler::digestCalcResponse(const THttpAuthHeader& hd, + const char* path, + const char* method, + const char* nonceCount, + char* outResponse, + char* outCNonce) { char HA1[33]; digestCalcHA1(hd, HA1, outCNonce); char HA2[33]; MD5 ctx; ctx.Init(); - addMD5(ctx, method); - addMD5Sep(ctx); - addMD5(ctx, path); + addMD5(ctx, method); + addMD5Sep(ctx); + addMD5(ctx, path); //ignore auth-int ctx.End(HA2); ctx.Init(); - addMD5(ctx, HA1, 32); - addMD5Sep(ctx); - addMD5(ctx, Nonce_); - addMD5Sep(ctx); + addMD5(ctx, HA1, 32); + addMD5Sep(ctx); + addMD5(ctx, Nonce_); + addMD5Sep(ctx); - if (hd.qop_auth) { + if (hd.qop_auth) { if (!*outCNonce) generateCNonce(outCNonce); - addMD5(ctx, nonceCount, 8); - addMD5Sep(ctx); - addMD5(ctx, outCNonce); - addMD5Sep(ctx); - addMD5(ctx, "auth", 4); - addMD5Sep(ctx); + addMD5(ctx, nonceCount, 8); + addMD5Sep(ctx); + addMD5(ctx, outCNonce); + addMD5Sep(ctx); + addMD5(ctx, "auth", 4); + addMD5Sep(ctx); } - addMD5(ctx, HA2, 32); + addMD5(ctx, HA2, 32); ctx.End(outResponse); } /************************************************************/ -bool httpDigestHandler::processHeader(const THttpAuthHeader* header, - const char* path, - const char* method, - const char* cnonce) { - if (!User_ || !header || !header->use_auth || !header->realm || !header->nonce) +bool httpDigestHandler::processHeader(const THttpAuthHeader* header, + const char* path, + const char* method, + const char* cnonce) { + if (!User_ || !header || !header->use_auth || !header->realm || !header->nonce) return false; - if (Nonce_) { - if (strcmp(Nonce_, header->nonce)) { - free(Nonce_); + if (Nonce_) { + if (strcmp(Nonce_, header->nonce)) { + free(Nonce_); Nonce_ = nullptr; - NonceCount_ = 0; + NonceCount_ = 0; } } - if (!Nonce_) { - Nonce_ = strdup(header->nonce); - NonceCount_ = 0; + if (!Nonce_) { + Nonce_ = strdup(header->nonce); + NonceCount_ = 0; } - free(HeaderInstruction_); + free(HeaderInstruction_); HeaderInstruction_ = nullptr; - NonceCount_++; + NonceCount_++; char nonceCount[20]; - sprintf(nonceCount, "%08d", NonceCount_); + sprintf(nonceCount, "%08d", NonceCount_); char CNonce[50]; if (cnonce) strcpy(CNonce, cnonce); else - CNonce[0] = 0; + CNonce[0] = 0; char response[33]; - digestCalcResponse(*header, path, method, nonceCount, response, CNonce); + digestCalcResponse(*header, path, method, nonceCount, response, CNonce); //digest-response = 1#( username | realm | nonce | digest-uri // | response | [ algorithm ] | [cnonce] | @@ -178,11 +178,11 @@ bool httpDigestHandler::processHeader(const THttpAuthHeader* header, // [nonce-count] | [auth-param] ) TStringStream out; - out << WWW_PREFIX << "username=\"" << User_ << "\""; + out << WWW_PREFIX << "username=\"" << User_ << "\""; out << ", realm=\"" << header->realm << "\""; out << ", nonce=\"" << header->nonce << "\""; out << ", uri=\"" << path << "\""; - if (header->algorithm == 1) + if (header->algorithm == 1) out << ", algorithm=MD5-sess"; else out << ", algorithm=MD5"; @@ -197,7 +197,7 @@ bool httpDigestHandler::processHeader(const THttpAuthHeader* header, out << "\r\n"; TString s_out = out.Str(); - HeaderInstruction_ = strdup(s_out.c_str()); + HeaderInstruction_ = strdup(s_out.c_str()); return true; } diff --git a/library/cpp/http/fetch/http_digest.h b/library/cpp/http/fetch/http_digest.h index 3b1872d70b..958737a6ac 100644 --- a/library/cpp/http/fetch/http_digest.h +++ b/library/cpp/http/fetch/http_digest.h @@ -1,46 +1,46 @@ #pragma once -#include "httpheader.h" - +#include "httpheader.h" + #include <util/system/compat.h> #include <library/cpp/http/misc/httpcodes.h> -class httpDigestHandler { -protected: - const char* User_; - const char* Password_; - char* Nonce_; - int NonceCount_; - char* HeaderInstruction_; +class httpDigestHandler { +protected: + const char* User_; + const char* Password_; + char* Nonce_; + int NonceCount_; + char* HeaderInstruction_; void clear(); - void generateCNonce(char* outCNonce); + void generateCNonce(char* outCNonce); - void digestCalcHA1(const THttpAuthHeader& hd, - char* outSessionKey, - char* outCNonce); + void digestCalcHA1(const THttpAuthHeader& hd, + char* outSessionKey, + char* outCNonce); - void digestCalcResponse(const THttpAuthHeader& hd, - const char* method, - const char* path, - const char* nonceCount, - char* outResponse, - char* outCNonce); + void digestCalcResponse(const THttpAuthHeader& hd, + const char* method, + const char* path, + const char* nonceCount, + char* outResponse, + char* outCNonce); -public: +public: httpDigestHandler(); ~httpDigestHandler(); - void setAuthorization(const char* user, - const char* password); - bool processHeader(const THttpAuthHeader* header, - const char* path, - const char* method, - const char* cnonce = nullptr); + void setAuthorization(const char* user, + const char* password); + bool processHeader(const THttpAuthHeader* header, + const char* path, + const char* method, + const char* cnonce = nullptr); - bool empty() const { - return (!User_); + bool empty() const { + return (!User_); } const char* getHeaderInstruction() const; diff --git a/library/cpp/http/fetch/http_socket.cpp b/library/cpp/http/fetch/http_socket.cpp index 1524ef04a8..7a5b29668c 100644 --- a/library/cpp/http/fetch/http_socket.cpp +++ b/library/cpp/http/fetch/http_socket.cpp @@ -7,7 +7,7 @@ #include <gcrypt.h> #include <gnutls/gnutls.h> -#include <util/network/init.h> +#include <util/network/init.h> #include <util/network/socket.h> #include <util/system/mutex.h> @@ -15,75 +15,75 @@ // HTTPS handler is used as implementation of // socketAbstractHandler for work through HTTPS protocol -class socketSecureHandler: public socketRegularHandler { -protected: - bool IsValid_; - gnutls_session Session_; - gnutls_certificate_credentials Credits_; - -public: - socketSecureHandler(); - virtual ~socketSecureHandler(); - - virtual bool Good(); - virtual int Connect(const TAddrList& addrs, TDuration Timeout); - virtual void Disconnect(); - virtual void shutdown(); - virtual bool send(const char* message, ssize_t messlen); - virtual bool peek(); - virtual ssize_t read(void* buffer, ssize_t buflen); +class socketSecureHandler: public socketRegularHandler { +protected: + bool IsValid_; + gnutls_session Session_; + gnutls_certificate_credentials Credits_; + +public: + socketSecureHandler(); + virtual ~socketSecureHandler(); + + virtual bool Good(); + virtual int Connect(const TAddrList& addrs, TDuration Timeout); + virtual void Disconnect(); + virtual void shutdown(); + virtual bool send(const char* message, ssize_t messlen); + virtual bool peek(); + virtual ssize_t read(void* buffer, ssize_t buflen); }; /********************************************************/ /********************************************************/ -static int gcry_pthread_mutex_init(void** priv) { +static int gcry_pthread_mutex_init(void** priv) { int err = 0; try { TMutex* lock = new TMutex; *priv = lock; - } catch (...) { + } catch (...) { err = -1; } return err; } -static int gcry_pthread_mutex_destroy(void** lock) { +static int gcry_pthread_mutex_destroy(void** lock) { delete static_cast<TMutex*>(*lock); - + return 0; } -static int gcry_pthread_mutex_lock(void** lock) { +static int gcry_pthread_mutex_lock(void** lock) { static_cast<TMutex*>(*lock)->Acquire(); return 0; } -static int gcry_pthread_mutex_unlock(void** lock) { +static int gcry_pthread_mutex_unlock(void** lock) { static_cast<TMutex*>(*lock)->Release(); return 0; } static struct gcry_thread_cbs gcry_threads_pthread = - { - GCRY_THREAD_OPTION_PTHREAD, NULL, - gcry_pthread_mutex_init, gcry_pthread_mutex_destroy, - gcry_pthread_mutex_lock, gcry_pthread_mutex_unlock, - NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL}; + { + GCRY_THREAD_OPTION_PTHREAD, NULL, + gcry_pthread_mutex_init, gcry_pthread_mutex_destroy, + gcry_pthread_mutex_lock, gcry_pthread_mutex_unlock, + NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL}; /********************************************************/ -struct https_initor { - https_initor() { - gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); +struct https_initor { + https_initor() { + gcry_control(GCRYCTL_SET_THREAD_CBS, &gcry_threads_pthread); gnutls_global_init(); - InitNetworkSubSystem(); + InitNetworkSubSystem(); } - ~https_initor() { + ~https_initor() { gnutls_global_deinit(); } }; @@ -91,112 +91,112 @@ struct https_initor { static https_initor _initor; /********************************************************/ -socketSecureHandler::socketSecureHandler() - : socketRegularHandler() - , IsValid_(false) - , Session_() - , Credits_() +socketSecureHandler::socketSecureHandler() + : socketRegularHandler() + , IsValid_(false) + , Session_() + , Credits_() { } /********************************************************/ -socketSecureHandler::~socketSecureHandler() { - if (IsValid_) +socketSecureHandler::~socketSecureHandler() { + if (IsValid_) Disconnect(); } /********************************************************/ -bool socketSecureHandler::Good() { - return Socket_.Good() && IsValid_; +bool socketSecureHandler::Good() { + return Socket_.Good() && IsValid_; } /********************************************************/ -int socketSecureHandler::Connect(const TAddrList& addrs, TDuration Timeout) { - IsValid_ = false; +int socketSecureHandler::Connect(const TAddrList& addrs, TDuration Timeout) { + IsValid_ = false; int ret = socketRegularHandler::Connect(addrs, Timeout); if (ret) return ret; - gnutls_certificate_allocate_credentials(&Credits_); - gnutls_init(&Session_, GNUTLS_CLIENT); - gnutls_set_default_priority(Session_); - gnutls_credentials_set(Session_, GNUTLS_CRD_CERTIFICATE, Credits_); + gnutls_certificate_allocate_credentials(&Credits_); + gnutls_init(&Session_, GNUTLS_CLIENT); + gnutls_set_default_priority(Session_); + gnutls_credentials_set(Session_, GNUTLS_CRD_CERTIFICATE, Credits_); - SOCKET fd = Socket_; - gnutls_transport_set_ptr(Session_, (gnutls_transport_ptr)fd); + SOCKET fd = Socket_; + gnutls_transport_set_ptr(Session_, (gnutls_transport_ptr)fd); - ret = gnutls_handshake(Session_); + ret = gnutls_handshake(Session_); - if (ret < 0) { + if (ret < 0) { fprintf(stderr, "*** Handshake failed\n"); gnutls_perror(ret); - gnutls_deinit(Session_); - if (Credits_) { - gnutls_certificate_free_credentials(Credits_); - Credits_ = 0; + gnutls_deinit(Session_); + if (Credits_) { + gnutls_certificate_free_credentials(Credits_); + Credits_ = 0; } return 1; } - IsValid_ = true; - return !IsValid_; + IsValid_ = true; + return !IsValid_; } /********************************************************/ -void socketSecureHandler::Disconnect() { - if (IsValid_) { - gnutls_bye(Session_, GNUTLS_SHUT_RDWR); - IsValid_ = false; - gnutls_deinit(Session_); +void socketSecureHandler::Disconnect() { + if (IsValid_) { + gnutls_bye(Session_, GNUTLS_SHUT_RDWR); + IsValid_ = false; + gnutls_deinit(Session_); } - if (Credits_) { - gnutls_certificate_free_credentials(Credits_); - Credits_ = 0; + if (Credits_) { + gnutls_certificate_free_credentials(Credits_); + Credits_ = 0; } socketRegularHandler::Disconnect(); } /********************************************************/ -void socketSecureHandler::shutdown() { +void socketSecureHandler::shutdown() { } /********************************************************/ -bool socketSecureHandler::send(const char* message, ssize_t messlen) { - if (!IsValid_) +bool socketSecureHandler::send(const char* message, ssize_t messlen) { + if (!IsValid_) return false; - ssize_t rv = gnutls_record_send(Session_, message, messlen); + ssize_t rv = gnutls_record_send(Session_, message, messlen); return rv >= 0; } /********************************************************/ -bool socketSecureHandler::peek() { +bool socketSecureHandler::peek() { //ssize_t rv = gnutls_record_check_pending(mSession); //return rv>0; return true; } /********************************************************/ -ssize_t socketSecureHandler::read(void* buffer, ssize_t buflen) { - if (!IsValid_) +ssize_t socketSecureHandler::read(void* buffer, ssize_t buflen) { + if (!IsValid_) return false; - return gnutls_record_recv(Session_, (char*)buffer, buflen); + return gnutls_record_recv(Session_, (char*)buffer, buflen); } #endif /************************************************************/ -socketAbstractHandler* socketHandlerFactory::chooseHandler(const THttpURL& url) { - if (url.IsValidGlobal() && url.GetScheme() == THttpURL::SchemeHTTP) +socketAbstractHandler* socketHandlerFactory::chooseHandler(const THttpURL& url) { + if (url.IsValidGlobal() && url.GetScheme() == THttpURL::SchemeHTTP) return new socketRegularHandler; -#ifdef USE_GNUTLS - if (url.IsValidGlobal() && url.GetScheme() == THttpURL::SchemeHTTPS) +#ifdef USE_GNUTLS + if (url.IsValidGlobal() && url.GetScheme() == THttpURL::SchemeHTTPS) return new socketSecureHandler; -#endif +#endif return nullptr; } diff --git a/library/cpp/http/fetch/httpfetcher.h b/library/cpp/http/fetch/httpfetcher.h index 7fc251afd2..4b02eef5fc 100644 --- a/library/cpp/http/fetch/httpfetcher.h +++ b/library/cpp/http/fetch/httpfetcher.h @@ -1,7 +1,7 @@ #pragma once #ifdef _MSC_VER -#include <io.h> +#include <io.h> #endif #include <library/cpp/http/misc/httpdate.h> @@ -10,27 +10,27 @@ #include "httpparser.h" struct TFakeBackup { - int Write(void* /*buf*/, size_t /*size*/) { - return 0; - } + int Write(void* /*buf*/, size_t /*size*/) { + return 0; + } }; template <size_t bufsize = 5000> struct TFakeAlloc { - void Shrink(void* /*buf*/, size_t /*size*/) { - } - void* Grab(size_t /*min*/, size_t* real) { - *real = bufsize; - return buf; - } - char buf[bufsize]; + void Shrink(void* /*buf*/, size_t /*size*/) { + } + void* Grab(size_t /*min*/, size_t* real) { + *real = bufsize; + return buf; + } + char buf[bufsize]; }; -template <typename TAlloc = TFakeAlloc<>, - typename TCheck = TFakeCheck<>, +template <typename TAlloc = TFakeAlloc<>, + typename TCheck = TFakeCheck<>, typename TWriter = TFakeBackup, - typename TAgent = THttpAgent<>> -class THttpFetcher: public THttpParser<TCheck>, public TAlloc, public TWriter, public TAgent { + typename TAgent = THttpAgent<>> +class THttpFetcher: public THttpParser<TCheck>, public TAlloc, public TWriter, public TAgent { public: static const size_t TCP_MIN = 1500; static int TerminateNow; @@ -40,21 +40,21 @@ public: , TAlloc() , TWriter() , TAgent() - { - } + { + } - virtual ~THttpFetcher() { - } + virtual ~THttpFetcher() { + } - int Fetch(THttpHeader* header, const char* path, const char* const* headers, int persistent, bool head_request = false) { + int Fetch(THttpHeader* header, const char* path, const char* const* headers, int persistent, bool head_request = false) { int ret = 0; int fetcherr = 0; THttpParser<TCheck>::Init(header, head_request); - const char* scheme = HttpUrlSchemeKindToString((THttpURL::TSchemeKind)TAgent::GetScheme()); + const char* scheme = HttpUrlSchemeKindToString((THttpURL::TSchemeKind)TAgent::GetScheme()); size_t schemelen = strlen(scheme); if (*path == '/') { - header->base = TStringBuf(scheme, schemelen); + header->base = TStringBuf(scheme, schemelen); header->base += TStringBuf("://", 3); header->base += TStringBuf(TAgent::pHostBeg, TAgent::pHostEnd - TAgent::pHostBeg); header->base += path; @@ -95,9 +95,9 @@ public: } if ((got = TAgent::read(bufptr, buffree)) < 0) { fetcherr = errno; - if (errno == EINTR) + if (errno == EINTR) header->error = HTTP_INTERRUPTED; - else if (errno == ETIMEDOUT) + else if (errno == ETIMEDOUT) header->error = HTTP_TIMEDOUT_WHILE_BYTES_RECEIVING; else header->error = HTTP_CONNECTION_LOST; @@ -113,7 +113,7 @@ public: THttpParser<TCheck>::Parse(parsebuf, got); if (header->error) - break; //if ANY error ocurred we will stop download that file or will have unprognosed stream position until MAX size reached + break; //if ANY error ocurred we will stop download that file or will have unprognosed stream position until MAX size reached if (inheader && THttpParser<TCheck>::GetState() != THttpParser<TCheck>::hp_in_header) { inheader = 0; @@ -141,12 +141,12 @@ public: i64 Adjustment = 0; if (!header->error) { - if (header->transfer_chunked) { + if (header->transfer_chunked) { Adjustment = header->header_size + header->entity_size - bufsize - 1; - } else if (header->content_length >= 0) { + } else if (header->content_length >= 0) { Adjustment = header->header_size + header->content_length - bufsize; } - if (Adjustment > 0) + if (Adjustment > 0) Adjustment = 0; } diff --git a/library/cpp/http/fetch/httpfsm.h b/library/cpp/http/fetch/httpfsm.h index c4abdcd0d2..af490adc5a 100644 --- a/library/cpp/http/fetch/httpfsm.h +++ b/library/cpp/http/fetch/httpfsm.h @@ -1,7 +1,7 @@ #pragma once -#include "httpheader.h" - +#include "httpheader.h" + #include <util/system/maxlen.h> #include <util/datetime/parser.h> @@ -15,7 +15,7 @@ struct THttpHeaderParser { static constexpr int NeedMore = 1; static constexpr int Accepted = 2; - int Execute(const void* inBuf, size_t len) { + int Execute(const void* inBuf, size_t len) { return execute((unsigned char*)inBuf, (int)len); } @@ -23,7 +23,7 @@ struct THttpHeaderParser { return Execute(str.data(), str.size()); } - int Init(THttpHeader* h) { + int Init(THttpHeader* h) { int ret = Init((THttpBaseHeader*)(h)); hd = h; hd->Init(); @@ -32,43 +32,43 @@ struct THttpHeaderParser { return ret; } - int Init(THttpAuthHeader* h) { + int Init(THttpAuthHeader* h) { int ret = Init((THttpHeader*)(h)); auth_hd = h; return ret; } - int Init(THttpRequestHeader* h) { + int Init(THttpRequestHeader* h) { int ret = Init((THttpBaseHeader*)(h)); request_hd = h; request_hd->Init(); return ret; } - THttpHeader* hd; - long I; - int Dc; + THttpHeader* hd; + long I; + int Dc; TDateTimeFieldsDeprecated DateTimeFields; - char buf[FETCHER_URL_MAX]; - size_t buflen; - char* lastchar; + char buf[FETCHER_URL_MAX]; + size_t buflen; + char* lastchar; - const unsigned char* langstart; - size_t langlen; + const unsigned char* langstart; + size_t langlen; - char* hreflangpos; - size_t hreflangspace; + char* hreflangpos; + size_t hreflangspace; bool AcceptingXRobots; - THttpAuthHeader* auth_hd; - THttpRequestHeader* request_hd; + THttpAuthHeader* auth_hd; + THttpRequestHeader* request_hd; private: - THttpBaseHeader* base_hd; + THttpBaseHeader* base_hd; int cs; - + private: - int Init(THttpBaseHeader* header) { + int Init(THttpBaseHeader* header) { base_hd = header; auth_hd = nullptr; request_hd = nullptr; @@ -77,12 +77,12 @@ private: return 0; } - int execute(unsigned char* inBuf, int len); + int execute(unsigned char* inBuf, int len); void init(); }; struct THttpChunkParser { - int Execute(const void* inBuf, int len) { + int Execute(const void* inBuf, int len) { return execute((unsigned char*)inBuf, len); } @@ -91,14 +91,14 @@ struct THttpChunkParser { return 0; } - int chunk_length; - char* lastchar; - long I; - int Dc; - i64 cnt64; + int chunk_length; + char* lastchar; + long I; + int Dc; + i64 cnt64; private: int cs; - int execute(unsigned char* inBuf, int len); + int execute(unsigned char* inBuf, int len); void init(); }; diff --git a/library/cpp/http/fetch/httpfsm_ut.cpp b/library/cpp/http/fetch/httpfsm_ut.cpp index b018e80101..d6393abd48 100644 --- a/library/cpp/http/fetch/httpfsm_ut.cpp +++ b/library/cpp/http/fetch/httpfsm_ut.cpp @@ -1,22 +1,22 @@ -#include "httpfsm.h" +#include "httpfsm.h" #include "library-htfetch_ut_hreflang_in.h" #include "library-htfetch_ut_hreflang_out.h" - + #include <util/generic/ptr.h> #include <library/cpp/charset/doccodes.h> #include <library/cpp/testing/unittest/registar.h> class THttpHeaderParserTestSuite: public TTestBase { UNIT_TEST_SUITE(THttpHeaderParserTestSuite); - UNIT_TEST(TestRequestHeader); - UNIT_TEST(TestSplitRequestHeader); - UNIT_TEST(TestTrailingData); - UNIT_TEST(TestProxyRequestHeader); - UNIT_TEST(TestIncorrectRequestHeader); - UNIT_TEST(TestLastModified); - UNIT_TEST(TestLastModifiedCorrupted); - UNIT_TEST(TestResponseHeaderOnRequest); - UNIT_TEST(TestRequestHeaderOnResponse); + UNIT_TEST(TestRequestHeader); + UNIT_TEST(TestSplitRequestHeader); + UNIT_TEST(TestTrailingData); + UNIT_TEST(TestProxyRequestHeader); + UNIT_TEST(TestIncorrectRequestHeader); + UNIT_TEST(TestLastModified); + UNIT_TEST(TestLastModifiedCorrupted); + UNIT_TEST(TestResponseHeaderOnRequest); + UNIT_TEST(TestRequestHeaderOnResponse); UNIT_TEST(TestXRobotsTagUnknownTags); UNIT_TEST(TestXRobotsTagMyBot); UNIT_TEST(TestXRobotsTagOtherBot); @@ -25,20 +25,20 @@ class THttpHeaderParserTestSuite: public TTestBase { UNIT_TEST(TestXRobotsTagOverridePriority); UNIT_TEST(TestXRobotsTagDoesNotBreakCharset); UNIT_TEST(TestXRobotsTagAllowsMultiline); - UNIT_TEST(TestRelCanonical); - UNIT_TEST(TestHreflang); - UNIT_TEST(TestHreflangOnLongInput); - UNIT_TEST(TestMimeType); + UNIT_TEST(TestRelCanonical); + UNIT_TEST(TestHreflang); + UNIT_TEST(TestHreflangOnLongInput); + UNIT_TEST(TestMimeType); UNIT_TEST(TestRepeatedContentEncoding); UNIT_TEST_SUITE_END(); - + private: THolder<THttpHeaderParser> httpHeaderParser; - + private: void TestStart(); void TestFinish(); - + public: void TestRequestHeader(); void TestSplitRequestHeader(); @@ -76,8 +76,8 @@ void THttpHeaderParserTestSuite::TestRequestHeader() { TestStart(); THttpRequestHeader httpRequestHeader; httpHeaderParser->Init(&httpRequestHeader); - const char* request = "GET /search?q=hi HTTP/1.1\r\n" - "Host: www.google.ru:8080\r\n\r\n"; + const char* request = "GET /search?q=hi HTTP/1.1\r\n" + "Host: www.google.ru:8080\r\n\r\n"; i32 result = httpHeaderParser->Execute(request, strlen(request)); UNIT_ASSERT_EQUAL(result, 2); UNIT_ASSERT_EQUAL(httpRequestHeader.http_method, HTTP_METHOD_GET); @@ -85,11 +85,11 @@ void THttpHeaderParserTestSuite::TestRequestHeader() { UNIT_ASSERT_EQUAL(httpRequestHeader.request_uri, "/search?q=hi"); UNIT_ASSERT_EQUAL(httpRequestHeader.GetUrl(), "http://www.google.ru:8080/search?q=hi"); UNIT_ASSERT_EQUAL(httpHeaderParser->lastchar - request + 1, - (i32)strlen(request)); + (i32)strlen(request)); UNIT_ASSERT_EQUAL(httpRequestHeader.x_yandex_response_timeout, - DEFAULT_RESPONSE_TIMEOUT); + DEFAULT_RESPONSE_TIMEOUT); UNIT_ASSERT_EQUAL(httpRequestHeader.x_yandex_request_priority, - DEFAULT_REQUEST_PRIORITY); + DEFAULT_REQUEST_PRIORITY); UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.x_yandex_sourcename, ""), 0); UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.x_yandex_requesttype, ""), 0); UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.x_yandex_fetchoptions, ""), 0); @@ -99,7 +99,7 @@ void THttpHeaderParserTestSuite::TestRequestHeader() { void THttpHeaderParserTestSuite::TestSplitRequestHeader() { TestStart(); - const char* request = + const char* request = "GET /search?q=hi HTTP/1.1\r\n" "Host: www.google.ru:8080 \r\n" "\r\n"; @@ -108,8 +108,8 @@ void THttpHeaderParserTestSuite::TestSplitRequestHeader() { for (size_t n1 = 0; n1 < rlen; n1++) { for (size_t n2 = n1; n2 < rlen; n2++) { TString s1{request, 0, n1}; - TString s2{request, n1, n2 - n1}; - TString s3{request, n2, rlen - n2}; + TString s2{request, n1, n2 - n1}; + TString s3{request, n2, rlen - n2}; UNIT_ASSERT_EQUAL(s1 + s2 + s3, request); THttpRequestHeader httpRequestHeader; @@ -134,7 +134,7 @@ void THttpHeaderParserTestSuite::TestTrailingData() { TestStart(); THttpRequestHeader httpRequestHeader; UNIT_ASSERT(0 == httpHeaderParser->Init(&httpRequestHeader)); - const char* request = + const char* request = "GET /search?q=hi HTTP/1.1\r\n" "Host: www.google.ru:8080\r\n" "\r\n" @@ -156,7 +156,7 @@ void THttpHeaderParserTestSuite::TestProxyRequestHeader() { TestStart(); THttpRequestHeader httpRequestHeader; httpHeaderParser->Init(&httpRequestHeader); - const char* request = + const char* request = "GET http://www.google.ru:8080/search?q=hi HTTP/1.1\r\n" "X-Yandex-Response-Timeout: 1000\r\n" "X-Yandex-Request-Priority: 2\r\n" @@ -179,16 +179,16 @@ void THttpHeaderParserTestSuite::TestProxyRequestHeader() { UNIT_ASSERT_VALUES_EQUAL(httpRequestHeader.if_modified_since, TInstant::ParseIso8601Deprecated("1994-10-29 19:43:31Z").TimeT()); UNIT_ASSERT_EQUAL(httpRequestHeader.request_uri, - "http://www.google.ru:8080/search?q=hi"); - UNIT_ASSERT(httpRequestHeader.GetUrl() == + "http://www.google.ru:8080/search?q=hi"); + UNIT_ASSERT(httpRequestHeader.GetUrl() == "http://www.google.ru:8080/search?q=hi"); UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.host, ""), 0); UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.from, "webadmin@yandex.ru"), 0); UNIT_ASSERT_EQUAL(strcmp(httpRequestHeader.user_agent, - "Yandex/1.01.001 (compatible; Win16; I)"), - 0); + "Yandex/1.01.001 (compatible; Win16; I)"), + 0); UNIT_ASSERT_EQUAL(httpHeaderParser->lastchar - request + 1, - (i32)strlen(request)); + (i32)strlen(request)); TestFinish(); } @@ -196,8 +196,8 @@ void THttpHeaderParserTestSuite::TestIncorrectRequestHeader() { TestStart(); THttpRequestHeader httpRequestHeader; httpHeaderParser->Init(&httpRequestHeader); - const char* request = "GET /search?q=hi HTP/1.1\r\n" - "Host: www.google.ru:8080\r\n\r\n"; + const char* request = "GET /search?q=hi HTP/1.1\r\n" + "Host: www.google.ru:8080\r\n\r\n"; i32 result = httpHeaderParser->Execute(request, strlen(request)); UNIT_ASSERT(result != 2); TestFinish(); @@ -214,7 +214,7 @@ void THttpHeaderParserTestSuite::TestLastModified() { UNIT_ASSERT(2 == httpHeaderParser->Execute(headers, strlen(headers))); UNIT_ASSERT_VALUES_EQUAL( TInstant::ParseIso8601Deprecated("2009-08-13 14:27:08Z").TimeT(), - h.http_time); + h.http_time); TestFinish(); } @@ -235,7 +235,7 @@ void THttpHeaderParserTestSuite::TestXRobotsTagUnknownTags() { TestStart(); THttpHeader httpHeader; httpHeaderParser->Init(&httpHeader); - const char* headers = + const char* headers = "HTTP/1.1 200 OK\r\n" "Content-Type: text/html\r\n" "x-robots-tag: asdfasdf asdf asdf,,, , noindex,noodpXXX , NOFOLLOW ,noodpnofollow\r\n\r\n"; @@ -384,7 +384,7 @@ void THttpHeaderParserTestSuite::TestHreflang() { TestStart(); THttpHeader httpHeader; httpHeaderParser->Init(&httpHeader); - const char* headers = + const char* headers = "HTTP/1.1 200 OK\r\n" "Content-Type: text/html\r\n" "link: <http://www.high.ru/>; rel='alternate'; hreflang='x-default'\r\n" @@ -414,7 +414,7 @@ void THttpHeaderParserTestSuite::TestRelCanonical() { TestStart(); THttpHeader httpHeader; httpHeaderParser->Init(&httpHeader); - const char* headers = + const char* headers = "HTTP/1.1 200 OK\r\n" "Content-Type: text/html\r\n" "Link: <http://yandex.ru>; rel = \"canonical\"\r\n\r\n"; @@ -428,8 +428,8 @@ void THttpHeaderParserTestSuite::TestResponseHeaderOnRequest() { TestStart(); THttpHeader httpHeader; httpHeaderParser->Init(&httpHeader); - const char* request = "GET /search?q=hi HTP/1.1\r\n" - "Host: www.google.ru:8080\r\n\r\n"; + const char* request = "GET /search?q=hi HTP/1.1\r\n" + "Host: www.google.ru:8080\r\n\r\n"; i32 result = httpHeaderParser->Execute(request, strlen(request)); UNIT_ASSERT_EQUAL(result, -3); TestFinish(); @@ -439,9 +439,9 @@ void THttpHeaderParserTestSuite::TestRequestHeaderOnResponse() { TestStart(); THttpRequestHeader httpRequestHeader; httpHeaderParser->Init(&httpRequestHeader); - const char* response = "HTTP/1.1 200 OK\r\n" - "Content-Type: text/html\r\n" - "Last-Modified: Thu, 13 Aug 2009 14:\r\n\r\n"; + const char* response = "HTTP/1.1 200 OK\r\n" + "Content-Type: text/html\r\n" + "Last-Modified: Thu, 13 Aug 2009 14:\r\n\r\n"; i32 result = httpHeaderParser->Execute(response, strlen(response)); UNIT_ASSERT_EQUAL(result, -3); TestFinish(); @@ -451,13 +451,13 @@ void THttpHeaderParserTestSuite::TestMimeType() { TestStart(); THttpHeader httpHeader; httpHeaderParser->Init(&httpHeader); - const char* headers = + const char* headers = "HTTP/1.1 200 OK\r\n" "Content-Type: application/json; charset=utf-8\r\n\r\n"; i32 result = httpHeaderParser->Execute(headers, strlen(headers)); UNIT_ASSERT_EQUAL(result, 2); UNIT_ASSERT_EQUAL(httpHeader.mime_type, static_cast<ui8>(MIME_JSON)); - UNIT_ASSERT_EQUAL(httpHeader.charset, static_cast<ui8>(CODES_UTF8)); + UNIT_ASSERT_EQUAL(httpHeader.charset, static_cast<ui8>(CODES_UTF8)); TestFinish(); } @@ -516,10 +516,10 @@ Y_UNIT_TEST_SUITE(TestHttpChunkParser) { Y_UNIT_TEST(TestWithoutEolHead) { const TStringBuf blob{ "4\r\n" - "____\r\n"}; + "____\r\n"}; TVector<int> states{ - -1, /* 1, -1, - 1, -1, 1, -1, 1, -1 */}; + -1, /* 1, -1, + 1, -1, 1, -1, 1, -1 */}; // as soon as error happens parser state should be considered // undefined, state is meaningless after the very first `-1` // moreover, testenv produces `states[1] == -1` for this input and @@ -530,7 +530,7 @@ Y_UNIT_TEST_SUITE(TestHttpChunkParser) { Y_UNIT_TEST(TestTrivialChunk) { const TStringBuf blob{ "\r\n" - "4\r\n"}; + "4\r\n"}; THttpChunkParser parser(parseBytesWithLastState(blob, 2)); UNIT_ASSERT_EQUAL(parser.chunk_length, 4); UNIT_ASSERT_EQUAL(parser.cnt64, 4); @@ -539,18 +539,18 @@ Y_UNIT_TEST_SUITE(TestHttpChunkParser) { Y_UNIT_TEST(TestNegative) { const TStringBuf blob{ "\r\n" - "-1"}; + "-1"}; TVector<int> states{ 1, 1, - -1, - /* 1 */}; + -1, + /* 1 */}; parseByteByByte(blob, states); } Y_UNIT_TEST(TestLeadingZero) { const TStringBuf blob{ "\r\n" - "042\r\n"}; + "042\r\n"}; THttpChunkParser parser(parseBytesWithLastState(blob, 2)); UNIT_ASSERT_EQUAL(parser.chunk_length, 0x42); } @@ -558,7 +558,7 @@ Y_UNIT_TEST_SUITE(TestHttpChunkParser) { Y_UNIT_TEST(TestIntOverflow) { const TStringBuf blob{ "\r\n" - "deadbeef"}; + "deadbeef"}; THttpChunkParser parser(parseBytesWithLastState(blob, -2)); UNIT_ASSERT_EQUAL(parser.chunk_length, 0); UNIT_ASSERT_EQUAL(parser.cnt64, 0xdeadbeef); @@ -571,9 +571,9 @@ Y_UNIT_TEST_SUITE(TestHttpChunkParser) { "_" // first byte of the chunk }; TVector<int> states{ - 1, 1, - 1, 1, 2, - -1}; + 1, 1, + 1, 1, 2, + -1}; parseByteByByte(blob, states); } @@ -584,7 +584,7 @@ Y_UNIT_TEST_SUITE(TestHttpChunkParser) { "\r\n" "000 ;foo = bar \r\n" "Trailer: bar\r\n" - "\r\n"}; + "\r\n"}; THttpChunkParser parser(parseBytesWithLastState(blob, 2)); UNIT_ASSERT_EQUAL(parser.chunk_length, 0); } diff --git a/library/cpp/http/fetch/httpheader.h b/library/cpp/http/fetch/httpheader.h index b2810bbd41..e779cb08ec 100644 --- a/library/cpp/http/fetch/httpheader.h +++ b/library/cpp/http/fetch/httpheader.h @@ -10,33 +10,33 @@ #include <util/generic/ylimits.h> #include <util/system/maxlen.h> -#include <ctime> +#include <ctime> #include <cstdio> #include <cstdlib> #include <cstring> -#include <algorithm> +#include <algorithm> // This is ugly solution but here a lot of work to do it the right way. #define FETCHER_URL_MAX 8192 -extern const i64 DEFAULT_RETRY_AFTER; /// == -1 +extern const i64 DEFAULT_RETRY_AFTER; /// == -1 extern const i64 DEFAULT_IF_MODIFIED_SINCE; /// == -1 -extern const i32 DEFAULT_MAX_AGE; /// == -1 -extern const i8 DEFAULT_REQUEST_PRIORITY; /// == -1 -extern const i32 DEFAULT_RESPONSE_TIMEOUT; /// == -1 +extern const i32 DEFAULT_MAX_AGE; /// == -1 +extern const i8 DEFAULT_REQUEST_PRIORITY; /// == -1 +extern const i32 DEFAULT_RESPONSE_TIMEOUT; /// == -1 #define HTTP_PREFIX "http://" #define MAX_LANGREGION_LEN 4 #define MAXWORD_LEN 55 enum HTTP_COMPRESSION { - HTTP_COMPRESSION_UNSET = 0, - HTTP_COMPRESSION_ERROR = 1, + HTTP_COMPRESSION_UNSET = 0, + HTTP_COMPRESSION_ERROR = 1, HTTP_COMPRESSION_IDENTITY = 2, - HTTP_COMPRESSION_GZIP = 3, - HTTP_COMPRESSION_DEFLATE = 4, + HTTP_COMPRESSION_GZIP = 3, + HTTP_COMPRESSION_DEFLATE = 4, HTTP_COMPRESSION_COMPRESS = 5, - HTTP_COMPRESSION_MAX = 6 + HTTP_COMPRESSION_MAX = 6 }; enum HTTP_METHOD { @@ -61,22 +61,22 @@ enum HTTP_CONNECTION { /// Class represents general http header fields. struct THttpBaseHeader { public: - i16 error; - i32 header_size; - i32 entity_size; - i64 content_length; - i64 http_time; // seconds since epoch - i64 content_range_start; // Content-Range: first-byte-pos - i64 content_range_end; // Content-Range: last-byte-pos - i64 content_range_entity_length; // Content-Range: entity-length - i8 http_minor; - i8 mime_type; - i8 charset; - i8 compression_method; - i8 transfer_chunked; - i8 connection_closed; + i16 error; + i32 header_size; + i32 entity_size; + i64 content_length; + i64 http_time; // seconds since epoch + i64 content_range_start; // Content-Range: first-byte-pos + i64 content_range_end; // Content-Range: last-byte-pos + i64 content_range_entity_length; // Content-Range: entity-length + i8 http_minor; + i8 mime_type; + i8 charset; + i8 compression_method; + i8 transfer_chunked; + i8 connection_closed; TString base; - + public: void Init() { error = 0; @@ -113,8 +113,8 @@ public: } int SetBase(const char* path, - const char* hostNamePtr = nullptr, - int hostNameLength = 0) { + const char* hostNamePtr = nullptr, + int hostNameLength = 0) { if (*path == '/') { base = "http://"; base += TStringBuf(hostNamePtr, hostNameLength); @@ -130,16 +130,16 @@ enum { HREFLANG_MAX = FETCHER_URL_MAX * 2 }; /// Class represents Http Response Header. struct THttpHeader: public THttpBaseHeader { public: - i8 accept_ranges; - i8 squid_error; + i8 accept_ranges; + i8 squid_error; i8 x_robots_tag; // deprecated, use x_robots_state instead - i16 http_status; + i16 http_status; TString location; TString rel_canonical; - char hreflangs[HREFLANG_MAX]; - i64 retry_after; + char hreflangs[HREFLANG_MAX]; + i64 retry_after; TString x_robots_state; // 'xxxxx' format, see `library/html/zoneconf/parsefunc.cpp` - + public: void Init() { THttpBaseHeader::Init(); @@ -174,12 +174,12 @@ public: char x_yandex_sourcename[MAXWORD_LEN]; char x_yandex_requesttype[MAXWORD_LEN]; char x_yandex_fetchoptions[MAXWORD_LEN]; - i8 http_method; - i8 x_yandex_request_priority; - i32 x_yandex_response_timeout; - i32 max_age; - i64 if_modified_since; - + i8 http_method; + i8 x_yandex_request_priority; + i32 x_yandex_response_timeout; + i32 max_age; + i64 if_modified_since; + public: THttpRequestHeader() { Init(); @@ -229,7 +229,7 @@ public: return url; } - char* GetUrl(char* buffer, size_t size) { + char* GetUrl(char* buffer, size_t size) { if (host[0] == 0 || !strcmp(host, "")) { strlcpy(buffer, request_uri.c_str(), size); } else { @@ -244,27 +244,27 @@ public: char* realm; char* nonce; char* opaque; - bool stale; - int algorithm; - bool qop_auth; - bool use_auth; + bool stale; + int algorithm; + bool qop_auth; + bool use_auth; //we do not provide auth-int variant as too heavy //bool qop_auth_int; - THttpAuthHeader() - : realm(nullptr) - , nonce(nullptr) - , opaque(nullptr) - , stale(false) - , algorithm(0) - , qop_auth(false) - , use_auth(true) + THttpAuthHeader() + : realm(nullptr) + , nonce(nullptr) + , opaque(nullptr) + , stale(false) + , algorithm(0) + , qop_auth(false) + , use_auth(true) { THttpHeader::Init(); } - ~THttpAuthHeader() { + ~THttpAuthHeader() { free(realm); free(nonce); free(opaque); @@ -272,7 +272,7 @@ public: void Print() { THttpHeader::Print(); - if (use_auth) { + if (use_auth) { if (realm) printf("realm: \"%s\"\n", realm); if (nonce) diff --git a/library/cpp/http/fetch/httpload.cpp b/library/cpp/http/fetch/httpload.cpp index 82ea8900b5..975a8d792c 100644 --- a/library/cpp/http/fetch/httpload.cpp +++ b/library/cpp/http/fetch/httpload.cpp @@ -2,210 +2,210 @@ /************************************************************/ /************************************************************/ -httpAgentReader::httpAgentReader(httpSpecialAgent& agent, - const char* baseUrl, - bool assumeConnectionClosed, - bool use_auth, - int bufSize) - : Header_() - , Agent_(agent) - , Buffer_(new char[bufSize]) - , BufPtr_(Buffer_) - , BufSize_(bufSize) - , BufRest_(0) +httpAgentReader::httpAgentReader(httpSpecialAgent& agent, + const char* baseUrl, + bool assumeConnectionClosed, + bool use_auth, + int bufSize) + : Header_() + , Agent_(agent) + , Buffer_(new char[bufSize]) + , BufPtr_(Buffer_) + , BufSize_(bufSize) + , BufRest_(0) { HeadRequest = false; - Header = &Header_; + Header = &Header_; if (use_auth) - HeaderParser.Init(&Header_); + HeaderParser.Init(&Header_); else HeaderParser.Init(Header); - setAssumeConnectionClosed(assumeConnectionClosed ? 1 : 0); - Header_.SetBase(baseUrl); + setAssumeConnectionClosed(assumeConnectionClosed ? 1 : 0); + Header_.SetBase(baseUrl); - if (Header_.error) + if (Header_.error) State = hp_error; else State = hp_in_header; } /************************************************************/ -httpAgentReader::~httpAgentReader() { - delete[] Buffer_; +httpAgentReader::~httpAgentReader() { + delete[] Buffer_; } /************************************************************/ -void httpAgentReader::readBuf() { - assert(BufRest_ == 0); - if (!BufPtr_) { - BufRest_ = -1; +void httpAgentReader::readBuf() { + assert(BufRest_ == 0); + if (!BufPtr_) { + BufRest_ = -1; return; } - BufRest_ = Agent_.read(Buffer_, BufSize_); - if (BufRest_ <= 0) { - BufRest_ = -1; - BufPtr_ = nullptr; + BufRest_ = Agent_.read(Buffer_, BufSize_); + if (BufRest_ <= 0) { + BufRest_ = -1; + BufPtr_ = nullptr; } else { - BufPtr_ = Buffer_; - + BufPtr_ = Buffer_; + //cout << "BUF: " << mBuffer << endl << endl; } } /************************************************************/ -const THttpHeader* httpAgentReader::readHeader() { - while (State == hp_in_header) { - if (!step()) { - Header_.error = HTTP_CONNECTION_LOST; +const THttpHeader* httpAgentReader::readHeader() { + while (State == hp_in_header) { + if (!step()) { + Header_.error = HTTP_CONNECTION_LOST; return nullptr; } - ParseGeneric(BufPtr_, BufRest_); + ParseGeneric(BufPtr_, BufRest_); } - if (State == hp_eof || State == hp_error) { + if (State == hp_eof || State == hp_error) { BufPtr_ = nullptr; - BufRest_ = -1; + BufRest_ = -1; } - if (State == hp_error || Header_.error) + if (State == hp_error || Header_.error) return nullptr; - return &Header_; + return &Header_; } /************************************************************/ -long httpAgentReader::readPortion(void*& buf) { - assert(State != hp_in_header); +long httpAgentReader::readPortion(void*& buf) { + assert(State != hp_in_header); long Chunk = 0; - do { - if (BufSize_ == 0 && !BufPtr_) + do { + if (BufSize_ == 0 && !BufPtr_) return 0; if (!step()) return 0; - Chunk = ParseGeneric(BufPtr_, BufRest_); - buf = BufPtr_; + Chunk = ParseGeneric(BufPtr_, BufRest_); + buf = BufPtr_; - if (State == hp_error && Header_.entity_size > Header_.content_length) { - Chunk -= (Header_.entity_size - Header_.content_length); - BufPtr_ = (char*)BufPtr_ + Chunk; - BufRest_ = 0; - State = hp_eof; - Header_.error = 0; + if (State == hp_error && Header_.entity_size > Header_.content_length) { + Chunk -= (Header_.entity_size - Header_.content_length); + BufPtr_ = (char*)BufPtr_ + Chunk; + BufRest_ = 0; + State = hp_eof; + Header_.error = 0; break; } - BufPtr_ = (char*)BufPtr_ + Chunk; - BufRest_ -= Chunk; + BufPtr_ = (char*)BufPtr_ + Chunk; + BufRest_ -= Chunk; - if (State == hp_eof || State == hp_error) { - BufRest_ = -1; - BufPtr_ = nullptr; + if (State == hp_eof || State == hp_error) { + BufRest_ = -1; + BufPtr_ = nullptr; } } while (!Chunk); return Chunk; } /************************************************************/ -bool httpAgentReader::skipTheRest() { +bool httpAgentReader::skipTheRest() { void* b; while (!eof()) readPortion(b); - return (State == hp_eof); + return (State == hp_eof); } /************************************************************/ /************************************************************/ -httpLoadAgent::httpLoadAgent(bool handleAuthorization, - socketHandlerFactory& factory) - : Factory_(factory) - , HandleAuthorization_(handleAuthorization) - , URL_() - , PersistentConn_(false) - , Reader_(nullptr) - , Headers_() - , ErrCode_(0) - , RealHost_(nullptr) +httpLoadAgent::httpLoadAgent(bool handleAuthorization, + socketHandlerFactory& factory) + : Factory_(factory) + , HandleAuthorization_(handleAuthorization) + , URL_() + , PersistentConn_(false) + , Reader_(nullptr) + , Headers_() + , ErrCode_(0) + , RealHost_(nullptr) { } /************************************************************/ -httpLoadAgent::~httpLoadAgent() { - delete Reader_; - free(RealHost_); +httpLoadAgent::~httpLoadAgent() { + delete Reader_; + free(RealHost_); } /************************************************************/ -void httpLoadAgent::clearReader() { - if (Reader_) { +void httpLoadAgent::clearReader() { + if (Reader_) { bool opened = false; - if (PersistentConn_) { - const THttpHeader* H = Reader_->readHeader(); - if (H && !H->connection_closed) { - Reader_->skipTheRest(); + if (PersistentConn_) { + const THttpHeader* H = Reader_->readHeader(); + if (H && !H->connection_closed) { + Reader_->skipTheRest(); opened = true; } } if (!opened) Disconnect(); - delete Reader_; + delete Reader_; Reader_ = nullptr; } - ErrCode_ = 0; + ErrCode_ = 0; } /************************************************************/ -void httpLoadAgent::setRealHost(const char* hostname) { - free(RealHost_); +void httpLoadAgent::setRealHost(const char* hostname) { + free(RealHost_); if (hostname) - RealHost_ = strdup(hostname); + RealHost_ = strdup(hostname); else RealHost_ = nullptr; - ErrCode_ = 0; + ErrCode_ = 0; } /************************************************************/ -void httpLoadAgent::setIMS(const char* ifModifiedSince) { +void httpLoadAgent::setIMS(const char* ifModifiedSince) { char ims_buf[100]; snprintf(ims_buf, 100, "If-Modified-Since: %s\r\n", ifModifiedSince); - Headers_.push_back(ims_buf); + Headers_.push_back(ims_buf); } /************************************************************/ -void httpLoadAgent::addHeaderInstruction(const char* instr) { - Headers_.push_back(instr); +void httpLoadAgent::addHeaderInstruction(const char* instr) { + Headers_.push_back(instr); } /************************************************************/ -void httpLoadAgent::dropHeaderInstructions() { - Headers_.clear(); +void httpLoadAgent::dropHeaderInstructions() { + Headers_.clear(); } /************************************************************/ -bool httpLoadAgent::startRequest(const THttpURL& url, - bool persistent, - const TAddrList& addrs) +bool httpLoadAgent::startRequest(const THttpURL& url, + bool persistent, + const TAddrList& addrs) { clearReader(); - ErrCode_ = 0; + ErrCode_ = 0; - URL_.Clear(); - URL_ = url; - PersistentConn_ = persistent; - if (!URL_.IsValidAbs()) + URL_.Clear(); + URL_ = url; + PersistentConn_ = persistent; + if (!URL_.IsValidAbs()) return false; - if (!HandleAuthorization_ && !URL_.IsNull(THttpURL::FlagAuth)) + if (!HandleAuthorization_ && !URL_.IsNull(THttpURL::FlagAuth)) return false; return doSetHost(addrs) && doStartRequest(); } /************************************************************/ -bool httpLoadAgent::startRequest(const char* url, - const char* url_to_merge, - bool persistent, - const TAddrList& addrs) { +bool httpLoadAgent::startRequest(const char* url, + const char* url_to_merge, + bool persistent, + const TAddrList& addrs) { clearReader(); URL_.Clear(); @@ -222,116 +222,116 @@ bool httpLoadAgent::startRequest(const char* url, } /************************************************************/ -bool httpLoadAgent::startRequest(const char* url, - const char* url_to_merge, - bool persistent, - ui32 ip) { +bool httpLoadAgent::startRequest(const char* url, + const char* url_to_merge, + bool persistent, + ui32 ip) { clearReader(); - URL_.Clear(); - PersistentConn_ = persistent; + URL_.Clear(); + PersistentConn_ = persistent; long flags = THttpURL::FeatureSchemeKnown | THttpURL::FeaturesNormalizeSet; - if (HandleAuthorization_) + if (HandleAuthorization_) flags |= THttpURL::FeatureAuthSupported; - if (URL_.Parse(url, flags, url_to_merge) || !URL_.IsValidGlobal()) + if (URL_.Parse(url, flags, url_to_merge) || !URL_.IsValidGlobal()) return false; return doSetHost(TAddrList::MakeV4Addr(ip, URL_.GetPort())) && doStartRequest(); } /************************************************************/ -bool httpLoadAgent::doSetHost(const TAddrList& addrs) { - socketAbstractHandler* h = Factory_.chooseHandler(URL_); +bool httpLoadAgent::doSetHost(const TAddrList& addrs) { + socketAbstractHandler* h = Factory_.chooseHandler(URL_); if (!h) return false; Socket.setHandler(h); - if (addrs.size()) { - ErrCode_ = SetHost(URL_.Get(THttpURL::FieldHost), + if (addrs.size()) { + ErrCode_ = SetHost(URL_.Get(THttpURL::FieldHost), URL_.GetPort(), addrs); } else { - ErrCode_ = SetHost(URL_.Get(THttpURL::FieldHost), - URL_.GetPort()); + ErrCode_ = SetHost(URL_.Get(THttpURL::FieldHost), + URL_.GetPort()); } - if (ErrCode_) + if (ErrCode_) return false; - if (RealHost_) { + if (RealHost_) { free(Hostheader); - Hostheader = (char*)malloc(strlen(RealHost_) + 20); - sprintf(Hostheader, "Host: %s\r\n", RealHost_); + Hostheader = (char*)malloc(strlen(RealHost_) + 20); + sprintf(Hostheader, "Host: %s\r\n", RealHost_); } - if (!URL_.IsNull(THttpURL::FlagAuth)) { - if (!HandleAuthorization_) { - ErrCode_ = HTTP_UNAUTHORIZED; + if (!URL_.IsNull(THttpURL::FlagAuth)) { + if (!HandleAuthorization_) { + ErrCode_ = HTTP_UNAUTHORIZED; return false; } - Digest_.setAuthorization(URL_.Get(THttpURL::FieldUsername), - URL_.Get(THttpURL::FieldPassword)); + Digest_.setAuthorization(URL_.Get(THttpURL::FieldUsername), + URL_.Get(THttpURL::FieldPassword)); } return true; } /************************************************************/ -bool httpLoadAgent::setHost(const char* host_url, - const TAddrList& addrs) { +bool httpLoadAgent::setHost(const char* host_url, + const TAddrList& addrs) { clearReader(); - URL_.Clear(); - PersistentConn_ = true; + URL_.Clear(); + PersistentConn_ = true; long flags = THttpURL::FeatureSchemeKnown | THttpURL::FeaturesNormalizeSet; - if (HandleAuthorization_) + if (HandleAuthorization_) flags |= THttpURL::FeatureAuthSupported; - if (URL_.Parse(host_url, flags) || !URL_.IsValidGlobal()) + if (URL_.Parse(host_url, flags) || !URL_.IsValidGlobal()) return false; return doSetHost(addrs); } /************************************************************/ -bool httpLoadAgent::startOneRequest(const char* local_url) { +bool httpLoadAgent::startOneRequest(const char* local_url) { clearReader(); THttpURL lURL; - if (lURL.Parse(local_url, THttpURL::FeaturesNormalizeSet) || lURL.IsValidGlobal()) + if (lURL.Parse(local_url, THttpURL::FeaturesNormalizeSet) || lURL.IsValidGlobal()) return false; - URL_.SetInMemory(THttpURL::FieldPath, lURL.Get(THttpURL::FieldPath)); - URL_.SetInMemory(THttpURL::FieldQuery, lURL.Get(THttpURL::FieldQuery)); - URL_.Rewrite(); + URL_.SetInMemory(THttpURL::FieldPath, lURL.Get(THttpURL::FieldPath)); + URL_.SetInMemory(THttpURL::FieldQuery, lURL.Get(THttpURL::FieldQuery)); + URL_.Rewrite(); return doStartRequest(); } /************************************************************/ -bool httpLoadAgent::doStartRequest() { - TString urlStr = URL_.PrintS(THttpURL::FlagPath | THttpURL::FlagQuery); +bool httpLoadAgent::doStartRequest() { + TString urlStr = URL_.PrintS(THttpURL::FlagPath | THttpURL::FlagQuery); if (!urlStr) urlStr = "/"; - for (int step = 0; step < 10; step++) { - const char* digestHeader = Digest_.getHeaderInstruction(); + for (int step = 0; step < 10; step++) { + const char* digestHeader = Digest_.getHeaderInstruction(); - unsigned i = (digestHeader) ? 2 : 1; + unsigned i = (digestHeader) ? 2 : 1; const char** headers = - (const char**)(alloca((i + Headers_.size()) * sizeof(char*))); + (const char**)(alloca((i + Headers_.size()) * sizeof(char*))); - for (i = 0; i < Headers_.size(); i++) - headers[i] = Headers_[i].c_str(); + for (i = 0; i < Headers_.size(); i++) + headers[i] = Headers_[i].c_str(); if (digestHeader) headers[i++] = digestHeader; - headers[i] = nullptr; + headers[i] = nullptr; - ErrCode_ = RequestGet(urlStr.c_str(), headers, PersistentConn_); + ErrCode_ = RequestGet(urlStr.c_str(), headers, PersistentConn_); - if (ErrCode_) { + if (ErrCode_) { Disconnect(); return false; } @@ -339,20 +339,20 @@ bool httpLoadAgent::doStartRequest() { TString urlBaseStr = URL_.PrintS(THttpURL::FlagNoFrag); clearReader(); - Reader_ = new httpAgentReader(*this, urlBaseStr.c_str(), - !PersistentConn_, !Digest_.empty()); + Reader_ = new httpAgentReader(*this, urlBaseStr.c_str(), + !PersistentConn_, !Digest_.empty()); - if (Reader_->readHeader()) { + if (Reader_->readHeader()) { //mReader->getHeader()->Print(); - if (getHeader()->http_status == HTTP_UNAUTHORIZED && - step < 1 && - Digest_.processHeader(getAuthHeader(), - urlStr.c_str(), - "GET")) { + if (getHeader()->http_status == HTTP_UNAUTHORIZED && + step < 1 && + Digest_.processHeader(getAuthHeader(), + urlStr.c_str(), + "GET")) { //mReader->skipTheRest(); - delete Reader_; + delete Reader_; Reader_ = nullptr; - ErrCode_ = 0; + ErrCode_ = 0; Disconnect(); continue; } @@ -365,7 +365,7 @@ bool httpLoadAgent::doStartRequest() { return false; } - ErrCode_ = HTTP_UNAUTHORIZED; + ErrCode_ = HTTP_UNAUTHORIZED; return false; } diff --git a/library/cpp/http/fetch/httpload.h b/library/cpp/http/fetch/httpload.h index e22e4b809e..5fc03d4be8 100644 --- a/library/cpp/http/fetch/httpload.h +++ b/library/cpp/http/fetch/httpload.h @@ -1,9 +1,9 @@ #pragma once -#include "httpagent.h" -#include "httpparser.h" -#include "http_digest.h" - +#include "httpagent.h" +#include "httpparser.h" +#include "http_digest.h" + #include <util/system/compat.h> #include <util/string/vector.h> #include <util/network/ip.h> @@ -20,7 +20,7 @@ /********************************************************/ // This class is used as a base one for flexible // socket handling -class socketAbstractHandler { +class socketAbstractHandler { public: virtual bool Good() = 0; @@ -36,12 +36,12 @@ public: virtual ssize_t read(void* buffer, ssize_t buflen) = 0; - virtual ~socketAbstractHandler() { - } + virtual ~socketAbstractHandler() { + } protected: - socketAbstractHandler() { - } + socketAbstractHandler() { + } }; /********************************************************/ @@ -49,54 +49,54 @@ protected: // socketAbstractHandler // (it is used by template scheme, // so it does not have virtual methods) -class TSocketHandlerPtr { +class TSocketHandlerPtr { protected: - socketAbstractHandler* Handler_; + socketAbstractHandler* Handler_; public: - TSocketHandlerPtr() - : Handler_(nullptr) - { - } + TSocketHandlerPtr() + : Handler_(nullptr) + { + } - virtual ~TSocketHandlerPtr() { - delete Handler_; - } + virtual ~TSocketHandlerPtr() { + delete Handler_; + } - int Good() { - return (Handler_ && Handler_->Good()); - } + int Good() { + return (Handler_ && Handler_->Good()); + } - int Connect(const TAddrList& addrs, TDuration Timeout) { - return (Handler_) ? Handler_->Connect(addrs, Timeout) : 1; + int Connect(const TAddrList& addrs, TDuration Timeout) { + return (Handler_) ? Handler_->Connect(addrs, Timeout) : 1; } - void Disconnect() { - if (Handler_) - Handler_->Disconnect(); + void Disconnect() { + if (Handler_) + Handler_->Disconnect(); } - void shutdown() { - if (Handler_) - Handler_->shutdown(); + void shutdown() { + if (Handler_) + Handler_->shutdown(); } - bool send(const char* message, ssize_t messlen) { - return (Handler_) ? Handler_->send(message, messlen) : false; + bool send(const char* message, ssize_t messlen) { + return (Handler_) ? Handler_->send(message, messlen) : false; } - virtual bool peek() { - return (Handler_) ? Handler_->peek() : false; + virtual bool peek() { + return (Handler_) ? Handler_->peek() : false; } - virtual ssize_t read(void* buffer, ssize_t buflen) { - return (Handler_) ? Handler_->read(buffer, buflen) : 0; + virtual ssize_t read(void* buffer, ssize_t buflen) { + return (Handler_) ? Handler_->read(buffer, buflen) : 0; } - void setHandler(socketAbstractHandler* handler) { - if (Handler_) - delete Handler_; - Handler_ = handler; + void setHandler(socketAbstractHandler* handler) { + if (Handler_) + delete Handler_; + Handler_ = handler; } }; @@ -108,58 +108,58 @@ using httpSpecialAgent = THttpAgent<TSocketHandlerPtr>; /********************************************************/ // Regular handler is used as implementation of // socketAbstractHandler for work through HTTP protocol -class socketRegularHandler: public socketAbstractHandler { -protected: - TSimpleSocketHandler Socket_; +class socketRegularHandler: public socketAbstractHandler { +protected: + TSimpleSocketHandler Socket_; -public: - socketRegularHandler() - : Socket_() - { - } +public: + socketRegularHandler() + : Socket_() + { + } - bool Good() override { - return Socket_.Good(); + bool Good() override { + return Socket_.Good(); } - int Connect(const TAddrList& addrs, TDuration Timeout) override { + int Connect(const TAddrList& addrs, TDuration Timeout) override { return Socket_.Connect(addrs, Timeout); } - void Disconnect() override { - Socket_.Disconnect(); + void Disconnect() override { + Socket_.Disconnect(); } - void shutdown() override { + void shutdown() override { //Do not block writing to socket //There are servers that works in a bad way with this //mSocket.shutdown(); } - bool send(const char* message, ssize_t messlen) override { - return Socket_.send(message, messlen); + bool send(const char* message, ssize_t messlen) override { + return Socket_.send(message, messlen); } - bool peek() override { - return Socket_.peek(); + bool peek() override { + return Socket_.peek(); } - ssize_t read(void* buffer, ssize_t buflen) override { - return Socket_.read(buffer, buflen); + ssize_t read(void* buffer, ssize_t buflen) override { + return Socket_.read(buffer, buflen); } }; /********************************************************/ // The base factory that allows to choose an appropriate // socketAbstractHandler implementation by url schema - -class socketHandlerFactory { -public: - virtual ~socketHandlerFactory() { - } + +class socketHandlerFactory { +public: + virtual ~socketHandlerFactory() { + } //returns mHandler_HTTP for correct HTTP-based url - virtual socketAbstractHandler* chooseHandler(const THttpURL& url); + virtual socketAbstractHandler* chooseHandler(const THttpURL& url); static socketHandlerFactory sInstance; }; @@ -168,138 +168,138 @@ public: // Section 2: the configurates tool to parse an HTTP-response /********************************************************/ -class httpAgentReader: public THttpParserGeneric<1> { -protected: - THttpAuthHeader Header_; - httpSpecialAgent& Agent_; +class httpAgentReader: public THttpParserGeneric<1> { +protected: + THttpAuthHeader Header_; + httpSpecialAgent& Agent_; - char* Buffer_; - void* BufPtr_; - int BufSize_; - long BufRest_; + char* Buffer_; + void* BufPtr_; + int BufSize_; + long BufRest_; void readBuf(); - bool step() { - if (BufRest_ == 0) + bool step() { + if (BufRest_ == 0) readBuf(); if (eof()) return false; return true; } -public: - httpAgentReader(httpSpecialAgent& agent, - const char* baseUrl, - bool assumeConnectionClosed, - bool use_auth = false, - int bufSize = 0x1000); +public: + httpAgentReader(httpSpecialAgent& agent, + const char* baseUrl, + bool assumeConnectionClosed, + bool use_auth = false, + int bufSize = 0x1000); ~httpAgentReader(); - bool eof() { - return BufRest_ < 0; + bool eof() { + return BufRest_ < 0; } - int error() { - return Header_.error; + int error() { + return Header_.error; } - void setError(int errCode) { - Header_.error = errCode; + void setError(int errCode) { + Header_.error = errCode; } - const THttpAuthHeader* getAuthHeader() { - return &Header_; + const THttpAuthHeader* getAuthHeader() { + return &Header_; } - const THttpHeader* readHeader(); - long readPortion(void*& buf); - bool skipTheRest(); + const THttpHeader* readHeader(); + long readPortion(void*& buf); + bool skipTheRest(); }; /********************************************************/ // Section 3: the main class /********************************************************/ -class httpLoadAgent: public httpSpecialAgent { -protected: - socketHandlerFactory& Factory_; - bool HandleAuthorization_; - THttpURL URL_; - bool PersistentConn_; - httpAgentReader* Reader_; - TVector<TString> Headers_; - int ErrCode_; - char* RealHost_; - httpDigestHandler Digest_; +class httpLoadAgent: public httpSpecialAgent { +protected: + socketHandlerFactory& Factory_; + bool HandleAuthorization_; + THttpURL URL_; + bool PersistentConn_; + httpAgentReader* Reader_; + TVector<TString> Headers_; + int ErrCode_; + char* RealHost_; + httpDigestHandler Digest_; void clearReader(); bool doSetHost(const TAddrList& addrs); bool doStartRequest(); -public: - httpLoadAgent(bool handleAuthorization = false, - socketHandlerFactory& factory = socketHandlerFactory::sInstance); +public: + httpLoadAgent(bool handleAuthorization = false, + socketHandlerFactory& factory = socketHandlerFactory::sInstance); ~httpLoadAgent(); void setRealHost(const char* host); - void setIMS(const char* ifModifiedSince); + void setIMS(const char* ifModifiedSince); void addHeaderInstruction(const char* instr); void dropHeaderInstructions(); bool startRequest(const char* url, const char* url_to_merge = nullptr, - bool persistent = false, + bool persistent = false, const TAddrList& addrs = TAddrList()); // deprecated v4-only bool startRequest(const char* url, const char* url_to_merge, - bool persistent, - ui32 ip); + bool persistent, + ui32 ip); bool startRequest(const THttpURL& url, - bool persistent = false, + bool persistent = false, const TAddrList& addrs = TAddrList()); - bool setHost(const char* host_url, - const TAddrList& addrs = TAddrList()); + bool setHost(const char* host_url, + const TAddrList& addrs = TAddrList()); bool startOneRequest(const char* local_url); - const THttpAuthHeader* getAuthHeader() { - if (Reader_ && Reader_->getAuthHeader()->use_auth) - return Reader_->getAuthHeader(); + const THttpAuthHeader* getAuthHeader() { + if (Reader_ && Reader_->getAuthHeader()->use_auth) + return Reader_->getAuthHeader(); return nullptr; } - const THttpHeader* getHeader() { - if (Reader_) - return Reader_->getAuthHeader(); + const THttpHeader* getHeader() { + if (Reader_) + return Reader_->getAuthHeader(); return nullptr; } - const THttpURL& getURL() { - return URL_; + const THttpURL& getURL() { + return URL_; } - bool eof() { - if (Reader_) - return Reader_->eof(); + bool eof() { + if (Reader_) + return Reader_->eof(); return true; } - int error() { - if (ErrCode_) - return ErrCode_; - if (Reader_) - return Reader_->error(); + int error() { + if (ErrCode_) + return ErrCode_; + if (Reader_) + return Reader_->error(); return HTTP_BAD_URL; } - long readPortion(void*& buf) { - if (Reader_) - return Reader_->readPortion(buf); + long readPortion(void*& buf) { + if (Reader_) + return Reader_->readPortion(buf); return -1; } }; diff --git a/library/cpp/http/fetch/httpparser.h b/library/cpp/http/fetch/httpparser.h index 769828e4ae..989409bdd2 100644 --- a/library/cpp/http/fetch/httpparser.h +++ b/library/cpp/http/fetch/httpparser.h @@ -6,22 +6,22 @@ #include <library/cpp/mime/types/mime.h> #include <util/system/yassert.h> #include <library/cpp/http/misc/httpcodes.h> - -template <size_t headermax = 100 << 10, size_t bodymax = 1 << 20> + +template <size_t headermax = 100 << 10, size_t bodymax = 1 << 20> struct TFakeCheck { - bool Check(THttpHeader* /*header*/) { - return false; - } - void CheckDocPart(void* /*buf*/, size_t /*len*/, THttpHeader* /*header*/) { - } //for every part of DocumentBody will be called - void CheckEndDoc(THttpHeader* /*header*/) { - } - size_t GetMaxHeaderSize() { - return headermax; - } - size_t GetMaxBodySize(THttpHeader*) { - return bodymax; - } + bool Check(THttpHeader* /*header*/) { + return false; + } + void CheckDocPart(void* /*buf*/, size_t /*len*/, THttpHeader* /*header*/) { + } //for every part of DocumentBody will be called + void CheckEndDoc(THttpHeader* /*header*/) { + } + size_t GetMaxHeaderSize() { + return headermax; + } + size_t GetMaxBodySize(THttpHeader*) { + return bodymax; + } }; class THttpParserBase { @@ -37,17 +37,17 @@ public: hp_read_chunk }; - States GetState() { - return State; - } + States GetState() { + return State; + } - void setAssumeConnectionClosed(int value) { + void setAssumeConnectionClosed(int value) { AssumeConnectionClosed = value; } - THttpHeader* GetHttpHeader() const { - return Header; - } + THttpHeader* GetHttpHeader() const { + return Header; + } protected: int CheckHeaders() { @@ -69,8 +69,8 @@ protected: if (Header->compression_method != HTTP_COMPRESSION_UNSET && Header->compression_method != HTTP_COMPRESSION_IDENTITY && Header->compression_method != HTTP_COMPRESSION_GZIP && - Header->compression_method != HTTP_COMPRESSION_DEFLATE) - { + Header->compression_method != HTTP_COMPRESSION_DEFLATE) + { Header->error = HTTP_BAD_CONTENT_ENCODING; return 1; } @@ -78,7 +78,7 @@ protected: if (Header->connection_closed == -1) Header->connection_closed = (Header->http_minor == 0 || AssumeConnectionClosed); - if (!Header->transfer_chunked && !Header->connection_closed && Header->content_length < 0 && !HeadRequest) { + if (!Header->transfer_chunked && !Header->connection_closed && Header->content_length < 0 && !HeadRequest) { Header->error = HTTP_LENGTH_UNKNOWN; return 1; } @@ -93,15 +93,15 @@ protected: THttpChunkParser ChunkParser; States State; long ChunkSize; - THttpHeader* Header; + THttpHeader* Header; int AssumeConnectionClosed; bool HeadRequest; }; -template <int isReader, typename TCheck = TFakeCheck<>> +template <int isReader, typename TCheck = TFakeCheck<>> class THttpParserGeneric: public THttpParserBase, public TCheck { protected: - long ParseGeneric(void*& buf, long& size) { + long ParseGeneric(void*& buf, long& size) { if (!size) { switch (State) { case hp_error: @@ -155,10 +155,10 @@ protected: size -= long(HeaderParser.lastchar - (char*)buf + 1); buf = HeaderParser.lastchar + 1; State = CheckHeaders() ? hp_error - : Header->transfer_chunked ? hp_begin_chunk_header - : Header->content_length == 0 ? hp_eof - : Header->content_length > 0 ? hp_read_alive - : hp_read_closed; + : Header->transfer_chunked ? hp_begin_chunk_header + : Header->content_length == 0 ? hp_eof + : Header->content_length > 0 ? hp_read_alive + : hp_read_closed; if (State == hp_begin_chunk_header) { // unget \n for chunk reader buf = (char*)buf - 1; @@ -239,11 +239,11 @@ protected: } }; -template <class TCheck = TFakeCheck<>> +template <class TCheck = TFakeCheck<>> class THttpParser: public THttpParserGeneric<0, TCheck> { typedef THttpParserGeneric<0, TCheck> TBaseT; //sorry avoiding gcc 3.4.6 BUG! public: - void Init(THttpHeader* H, bool head_request = false) { + void Init(THttpHeader* H, bool head_request = false) { TBaseT::Header = H; TBaseT::HeaderParser.Init(TBaseT::Header); TBaseT::State = TBaseT::hp_in_header; @@ -251,19 +251,19 @@ public: TBaseT::HeadRequest = head_request; } - void Parse(void* buf, long size) { + void Parse(void* buf, long size) { TBaseT::ParseGeneric(buf, size); } }; class TMemoReader { public: - int Init(void* buf, long bufsize) { + int Init(void* buf, long bufsize) { Buf = buf; Bufsize = bufsize; return 0; } - long Read(void*& buf) { + long Read(void*& buf) { Y_ASSERT(Bufsize >= 0); if (!Bufsize) { Bufsize = -1; @@ -277,15 +277,15 @@ public: protected: long Bufsize; - void* Buf; + void* Buf; }; template <class Reader> class THttpReader: public THttpParserGeneric<1>, public Reader { typedef THttpParserGeneric<1> TBaseT; - + public: - using TBaseT::AssumeConnectionClosed; + using TBaseT::AssumeConnectionClosed; using TBaseT::Header; using TBaseT::ParseGeneric; using TBaseT::State; @@ -299,7 +299,7 @@ public: return parsHeader ? ParseHeader() : SkipHeader(); } - long Read(void*& buf) { + long Read(void*& buf) { long Chunk; do { if (!Size) { @@ -317,7 +317,7 @@ public: if (State == hp_eof) { Size = 0; Eoferr = 0; - } else if (State == hp_error) + } else if (State == hp_error) return Eoferr = -1; } while (!Chunk); return Chunk; @@ -352,9 +352,9 @@ protected: hdrsize -= Size; } State = Header->transfer_chunked ? hp_begin_chunk_header - : Header->content_length == 0 ? hp_eof - : Header->content_length > 0 ? hp_read_alive - : hp_read_closed; + : Header->content_length == 0 ? hp_eof + : Header->content_length > 0 ? hp_read_alive + : hp_read_closed; Header->entity_size = 0; if (State == hp_eof) Eoferr = 0; @@ -366,7 +366,7 @@ protected: return 0; } - void* Ptr; + void* Ptr; long Size; - int Eoferr; + int Eoferr; }; diff --git a/library/cpp/http/fetch/httpparser_ut.cpp b/library/cpp/http/fetch/httpparser_ut.cpp index 3b3b938e7a..6957eb2f5f 100644 --- a/library/cpp/http/fetch/httpparser_ut.cpp +++ b/library/cpp/http/fetch/httpparser_ut.cpp @@ -2,13 +2,13 @@ #include <library/cpp/testing/unittest/registar.h> -#define ENUM_OUT(arg) \ - case type ::arg: { \ - out << #arg; \ - return; \ - } +#define ENUM_OUT(arg) \ + case type ::arg: { \ + out << #arg; \ + return; \ + } -template <> +template <> void Out<THttpParserBase::States>(IOutputStream& out, THttpParserBase::States st) { using type = THttpParserBase::States; switch (st) { @@ -24,27 +24,27 @@ void Out<THttpParserBase::States>(IOutputStream& out, THttpParserBase::States st } namespace { - class TSomethingLikeFakeCheck; + class TSomethingLikeFakeCheck; - using TTestHttpParser = THttpParser<TSomethingLikeFakeCheck>; + using TTestHttpParser = THttpParser<TSomethingLikeFakeCheck>; - class TSomethingLikeFakeCheck { - TString Body_; + class TSomethingLikeFakeCheck { + TString Body_; - public: - const TString& Body() const { - return Body_; - } + public: + const TString& Body() const { + return Body_; + } - // other functions are not really called by THttpParser - void CheckDocPart(const void* buf, size_t len, THttpHeader* /* header */) { - TString s(static_cast<const char*>(buf), len); - Cout << "State = " << static_cast<TTestHttpParser*>(this)->GetState() << ", CheckDocPart(" << s.Quote() << ")\n"; - Body_ += s; - } - }; + // other functions are not really called by THttpParser + void CheckDocPart(const void* buf, size_t len, THttpHeader* /* header */) { + TString s(static_cast<const char*>(buf), len); + Cout << "State = " << static_cast<TTestHttpParser*>(this)->GetState() << ", CheckDocPart(" << s.Quote() << ")\n"; + Body_ += s; + } + }; -} +} Y_UNIT_TEST_SUITE(TestHttpParser) { Y_UNIT_TEST(TestTrivialRequest) { @@ -73,9 +73,9 @@ Y_UNIT_TEST_SUITE(TestHttpParser) { UNIT_ASSERT_EQUAL(parser.GetState(), parser.hp_eof); UNIT_ASSERT_EQUAL(parser.Body(), "OK"); UNIT_ASSERT_EQUAL(hdr.header_size, strlen( - "HTTP/1.1 200 Ok\r\n" - "Content-Length: 2\r\n" - "\r\n")); + "HTTP/1.1 200 Ok\r\n" + "Content-Length: 2\r\n" + "\r\n")); UNIT_ASSERT_EQUAL(hdr.entity_size, strlen("OK")); } @@ -98,36 +98,36 @@ Y_UNIT_TEST_SUITE(TestHttpParser) { UNIT_ASSERT_EQUAL(parser.GetState(), parser.hp_eof); UNIT_ASSERT_EQUAL(parser.Body(), "OkAllRight"); UNIT_ASSERT_EQUAL(hdr.header_size, strlen( - "HTTP/1.1 200 OK\r\n" - "Transfer-Encoding: chunked\r\n" - "\r\n")); + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n")); const int off_by_one_err = -1; // XXX: it really looks so UNIT_ASSERT_EQUAL(hdr.entity_size + off_by_one_err, strlen( - "2\r\n" - "Ok\r\n" - "8\r\n" - "AllRight\r\n" - "0\r\n" - "\r\n")); + "2\r\n" + "Ok\r\n" + "8\r\n" + "AllRight\r\n" + "0\r\n" + "\r\n")); } static const TString PipelineClenBlob_{ - "HTTP/1.1 200 Ok\r\n" - "Content-Length: 4\r\n" - "\r\n" - "OK\r\n" - "HTTP/1.1 200 Zz\r\n" - "Content-Length: 4\r\n" - "\r\n" - "ZZ\r\n"}; - - void AssertPipelineClen(TTestHttpParser & parser, const THttpHeader& hdr) { + "HTTP/1.1 200 Ok\r\n" + "Content-Length: 4\r\n" + "\r\n" + "OK\r\n" + "HTTP/1.1 200 Zz\r\n" + "Content-Length: 4\r\n" + "\r\n" + "ZZ\r\n"}; + + void AssertPipelineClen(TTestHttpParser & parser, const THttpHeader& hdr) { UNIT_ASSERT_EQUAL(parser.GetState(), parser.hp_eof); UNIT_ASSERT_EQUAL(4, hdr.content_length); UNIT_ASSERT_EQUAL(hdr.header_size, strlen( - "HTTP/1.1 200 Ok\r\n" - "Content-Length: 4\r\n" - "\r\n")); + "HTTP/1.1 200 Ok\r\n" + "Content-Length: 4\r\n" + "\r\n")); } Y_UNIT_TEST(TestPipelineClenByteByByte) { @@ -154,55 +154,55 @@ Y_UNIT_TEST_SUITE(TestHttpParser) { parser.Parse((void*)blob.data(), blob.size()); AssertPipelineClen(parser, hdr); UNIT_ASSERT_EQUAL(parser.Body(), - "OK\r\n" - "HTTP/1.1 200 Zz\r\n" - "Content-Length: 4\r\n" - "\r\n" - "ZZ\r\n"); + "OK\r\n" + "HTTP/1.1 200 Zz\r\n" + "Content-Length: 4\r\n" + "\r\n" + "ZZ\r\n"); UNIT_ASSERT_EQUAL(hdr.entity_size, strlen( - "OK\r\n" - "HTTP/1.1 200 Zz\r\n" - "Content-Length: 4\r\n" - "\r\n" - "ZZ\r\n")); + "OK\r\n" + "HTTP/1.1 200 Zz\r\n" + "Content-Length: 4\r\n" + "\r\n" + "ZZ\r\n")); } static const TString PipelineChunkedBlob_{ - "HTTP/1.1 200 OK\r\n" - "Transfer-Encoding: chunked\r\n" - "\r\n" - "2\r\n" - "Ok\r\n" - "8\r\n" - "AllRight\r\n" - "0\r\n" - "\r\n" - "HTTP/1.1 200 OK\r\n" - "Transfer-Encoding: chunked\r\n" - "\r\n" - "2\r\n" - "Yo\r\n" - "8\r\n" - "uWin!Iam\r\n" - "0\r\n" - "\r\n"}; - - void AssertPipelineChunked(TTestHttpParser & parser, const THttpHeader& hdr) { + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "2\r\n" + "Ok\r\n" + "8\r\n" + "AllRight\r\n" + "0\r\n" + "\r\n" + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "2\r\n" + "Yo\r\n" + "8\r\n" + "uWin!Iam\r\n" + "0\r\n" + "\r\n"}; + + void AssertPipelineChunked(TTestHttpParser & parser, const THttpHeader& hdr) { UNIT_ASSERT_EQUAL(parser.GetState(), parser.hp_eof); UNIT_ASSERT_EQUAL(parser.Body(), "OkAllRight"); UNIT_ASSERT_EQUAL(-1, hdr.content_length); UNIT_ASSERT_EQUAL(hdr.header_size, strlen( - "HTTP/1.1 200 OK\r\n" - "Transfer-Encoding: chunked\r\n" - "\r\n")); + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n")); const int off_by_one_err = -1; UNIT_ASSERT_EQUAL(hdr.entity_size + off_by_one_err, strlen( - "2\r\n" - "Ok\r\n" - "8\r\n" - "AllRight\r\n" - "0\r\n" - "\r\n")); + "2\r\n" + "Ok\r\n" + "8\r\n" + "AllRight\r\n" + "0\r\n" + "\r\n")); } Y_UNIT_TEST(TestPipelineChunkedByteByByte) { diff --git a/library/cpp/http/fetch/httpzreader.h b/library/cpp/http/fetch/httpzreader.h index 68eb00853d..a8df608a9f 100644 --- a/library/cpp/http/fetch/httpzreader.h +++ b/library/cpp/http/fetch/httpzreader.h @@ -1,26 +1,26 @@ #pragma once -#include "httpheader.h" -#include "httpparser.h" -#include "exthttpcodes.h" +#include "httpheader.h" +#include "httpparser.h" +#include "exthttpcodes.h" #include <util/system/defaults.h> #include <util/generic/yexception.h> #include <contrib/libs/zlib/zlib.h> -#include <errno.h> - +#include <errno.h> + #ifndef ENOTSUP -#define ENOTSUP 45 +#define ENOTSUP 45 #endif template <class Reader> class TCompressedHttpReader: public THttpReader<Reader> { typedef THttpReader<Reader> TBase; - + public: - using TBase::AssumeConnectionClosed; + using TBase::AssumeConnectionClosed; using TBase::Header; using TBase::ParseGeneric; using TBase::State; @@ -78,7 +78,7 @@ public: return ret; } - long Read(void*& buf) { + long Read(void*& buf) { if (!CompressedInput) { long res = TBase::Read(buf); if (res > 0) { @@ -93,7 +93,7 @@ public: while (true) { if (Stream.avail_in == 0) { - void* tmpin = Stream.next_in; + void* tmpin = Stream.next_in; long res = TBase::Read(tmpin); Stream.next_in = (Bytef*)tmpin; if (res <= 0) @@ -121,10 +121,10 @@ public: ZErr = E2BIG; return -1; } - if (!IgnoreTrailingGarbage && BufSize == Stream.avail_out && Stream.avail_in > 0) { + if (!IgnoreTrailingGarbage && BufSize == Stream.avail_out && Stream.avail_in > 0) { Header->error = EXT_HTTP_GZIPERROR; ZErr = EFAULT; - Stream.msg = (char*)"trailing garbage"; + Stream.msg = (char*)"trailing garbage"; return -1; } return long(BufSize - Stream.avail_out); @@ -148,7 +148,7 @@ public: return -1; } - const char* ZMsg() const { + const char* ZMsg() const { return Stream.msg; } @@ -173,7 +173,7 @@ protected: return 0; case HTTP_COMPRESSION_GZIP: CompressedInput = true; - winsize += 16; // 16 indicates gzip, see zlib.h + winsize += 16; // 16 indicates gzip, see zlib.h break; case HTTP_COMPRESSION_DEFLATE: CompressedInput = true; @@ -240,21 +240,21 @@ protected: bool IgnoreTrailingGarbage; }; -class zlib_exception: public yexception { +class zlib_exception: public yexception { }; template <class Reader> class SCompressedHttpReader: public TCompressedHttpReader<Reader> { typedef TCompressedHttpReader<Reader> TBase; - + public: using TBase::ZError; using TBase::ZMsg; SCompressedHttpReader() - : TBase() - { - } + : TBase() + { + } int Init( THttpHeader* H, @@ -268,7 +268,7 @@ public: return (int)HandleRetValue((long)ret); } - long Read(void*& buf) { + long Read(void*& buf) { long ret = TBase::Read(buf); return HandleRetValue(ret); } @@ -281,7 +281,7 @@ protected: case ENOMEM: ythrow yexception() << "SCompressedHttpReader: not enough memory"; case EINVAL: - ythrow yexception() << "SCompressedHttpReader: zlib error: " << ZMsg(); + ythrow yexception() << "SCompressedHttpReader: zlib error: " << ZMsg(); case ENOTSUP: ythrow yexception() << "SCompressedHttpReader: unsupported compression method"; case EFAULT: diff --git a/library/cpp/http/fetch/library-htfetch_ut_hreflang_in.h b/library/cpp/http/fetch/library-htfetch_ut_hreflang_in.h index 0df89bdc79..d01a5911e7 100644 --- a/library/cpp/http/fetch/library-htfetch_ut_hreflang_in.h +++ b/library/cpp/http/fetch/library-htfetch_ut_hreflang_in.h @@ -1,155 +1,155 @@ #pragma once char hreflang_ut_in[] = "HTTP/1.1 200 OK\n" - "Date: Thu, 15 Nov 2012 22:38:28 GMT\n" - "Server: Apache/2\n" - "X-Powered-By: PHP/5.2.17\n" - "Set-Cookie: PHPSESSID=6d69474d1cc019d7d82714c9472bc6d6; path=/\n" - "Expires: Thu, 19 Nov 1981 08:52:00 GMT\n" - "Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0\n" - "Pragma: no-cache\n" - "Link: <http://www.forexticket.cn.com/zh/currency/converter-EEK-XAG>; rel='alternate'; hreflang='zh-CN'\n" - "Link: <http://www.forexticket.tw/zh/currency/converter-EEK-XAG>; rel='alternate'; hreflang='zh-TW'\n" - "Link: <http://www.forexticket.hk/zh/currency/converter-EEK-XAG>; rel='alternate'; hreflang='zh-HK'\n" - "Link: <http://www.forexticket.sg/zh/currency/converter-EEK-XAG>; rel='alternate'; hreflang='zh-SG'\n" - "Link: <http://www.forexticket.in/hi/currency/converter-EEK-XAG>; rel='alternate'; hreflang='hi-IN'\n" - "Link: <http://www.forexticket.com.fj/hi/currency/converter-EEK-XAG>; rel='alternate'; hreflang='hi-FJ'\n" - "Link: <http://www.forexticket.in/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-IN'\n" - "Link: <http://www.forexticket.us/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-US'\n" - "Link: <http://www.forexticket.com.pk/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-PK'\n" - "Link: <http://www.forexticket-bd.com/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-BD'\n" - "Link: <http://www.forexticket-ng.com/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-NG'\n" - "Link: <http://www.forexticket.co.uk/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-GB'\n" - "Link: <http://www.forexticket.co.za/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-ZA'\n" - "Link: <http://www.forexticket.co.ke/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-KE'\n" - "Link: <http://www.forexticket.com/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-CA'\n" - "Link: <http://www.forexticket-gh.com/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-GH'\n" - "Link: <http://www.forexticket.biz/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-AU'\n" - "Link: <http://www.forexticket.cm/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-CM'\n" - "Link: <http://www.forexticket-kh.com/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-KH'\n" - "Link: <http://www.forexticket.hk/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-HK'\n" - "Link: <http://www.forexticket.la/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-LA'\n" - "Link: <http://www.forexticket.sg/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-SG'\n" - "Link: <http://www.forexticket.co.nz/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-NZ'\n" - "Link: <http://www.forexticket.com.pr/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-PR'\n" - "Link: <http://www.forexticket.com.fj/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-FJ'\n" - "Link: <http://www.forexticket.us/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-US'\n" - "Link: <http://www.forexticket.mx/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-MX'\n" - "Link: <http://www.forexticket.co/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-CO'\n" - "Link: <http://www.forexticket.com.ar/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-AR'\n" - "Link: <http://www.forexticket-pe.com/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-PE'\n" - "Link: <http://www.forexticket.co.ve/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-VE'\n" - "Link: <http://www.forexticket.cl/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-CL'\n" - "Link: <http://www.forexticket.ec/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-EC'\n" - "Link: <http://www.forexticket.com.gt/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-GT'\n" - "Link: <http://www.forexticket.bo/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-BO'\n" - "Link: <http://www.forexticket.hn/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-HN'\n" - "Link: <http://www.forexticket.com.py/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-PY'\n" - "Link: <http://www.forexticket.es/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-ES'\n" - "Link: <http://www.forexticket.com.sv/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-SV'\n" - "Link: <http://www.forexticket.com.ni/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-NI'\n" - "Link: <http://www.forexticket.co.cr/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-CR'\n" - "Link: <http://www.forexticket.com.pr/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-PR'\n" - "Link: <http://www.forexticket.com.uy/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-UY'\n" - "Link: <http://www.forexticket.com.pa/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-PA'\n" - "Link: <http://www.forexticket.asia.com/id/currency/converter-EEK-XAG>; rel='alternate'; hreflang='id-ID'\n" - "Link: <http://www.forexticket.com.br/pt/moeda/conversor-EEK-XAG>; rel='alternate'; hreflang='pt-BR'\n" - "Link: <http://www.forexticket-mz.com/pt/moeda/conversor-EEK-XAG>; rel='alternate'; hreflang='pt-MZ'\n" - "Link: <http://www.forexticket.com.pt/pt/moeda/conversor-EEK-XAG>; rel='alternate'; hreflang='pt-PT'\n" - "Link: <http://www.forexticket.tl/pt/moeda/conversor-EEK-XAG>; rel='alternate'; hreflang='pt-TL'\n" - "Link: <http://www.forexticket.ru/ru/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ru-RU'\n" - "Link: <http://www.forexticket-kz.com/ru/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ru-KZ'\n" - "Link: <http://www.forexticket-tj.com/ru/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ru-TJ'\n" - "Link: <http://www.forexticket-kg.com/ru/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ru-KG'\n" - "Link: <http://www.forexticket-ge.com/ru/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ru-GE'\n" - "Link: <http://www.forexticket.mn/ru/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ru-MN'\n" - "Link: <http://www.forexticket.jp/ja/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ja-JP'\n" - "Link: <http://www.forexticket-ph.com/tl/currency/converter-EEK-XAG>; rel='alternate'; hreflang='tl-PH'\n" - "Link: <http://www.forexticket.vn/vi/currency/converter-EEK-XAG>; rel='alternate'; hreflang='vi-VN'\n" - "Link: <http://www.forexticket.de/de/waehrungsumrechner/devisen-EEK-XAG>; rel='alternate'; hreflang='de-DE'\n" - "Link: <http://www.forexticket.be/de/waehrungsumrechner/devisen-EEK-XAG>; rel='alternate'; hreflang='de-BE'\n" - "Link: <http://www.forexticket.at/de/waehrungsumrechner/devisen-EEK-XAG>; rel='alternate'; hreflang='de-AT'\n" - "Link: <http://www.forexticket.ch/de/waehrungsumrechner/devisen-EEK-XAG>; rel='alternate'; hreflang='de-CH'\n" - "Link: <http://www.forexticket.lu/de/waehrungsumrechner/devisen-EEK-XAG>; rel='alternate'; hreflang='de-LU'\n" - "Link: <http://www.forexticket.li/de/waehrungsumrechner/devisen-EEK-XAG>; rel='alternate'; hreflang='de-LI'\n" - "Link: <http://www.forexticket.de/de/waehrungsumrechner/devisen-EEK-XAG>; rel='canonical'\n" - "Link: <http://www.forexticket-eg.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-EG'\n" - "Link: <http://www.forexticket-dz.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-DZ'\n" - "Link: <http://www.forexticket-ma.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-MA'\n" - "Link: <http://www.forexticket-iq.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-IQ'\n" - "Link: <http://www.forexticket-sa.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-SA'\n" - "Link: <http://www.forexticket-sy.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-SY'\n" - "Link: <http://www.forexticket-tn.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-TN'\n" - "Link: <http://www.forexticket-td.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-TD'\n" - "Link: <http://www.forexticket-so.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-SO'\n" - "Link: <http://www.forexticket.co.il/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-IL'\n" - "Link: <http://www.forexticket-jo.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-JO'\n" - "Link: <http://www.forexticket.ae/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-AE'\n" - "Link: <http://www.forexticket-lb.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-LB'\n" - "Link: <http://www.forexticket-om.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-OM'\n" - "Link: <http://www.forexticket-kw.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-KW'\n" - "Link: <http://www.forexticket-tr.com/tr/currency/converter-EEK-XAG>; rel='alternate'; hreflang='tr-TR'\n" - "Link: <http://www.forexticket-bg.com/tr/currency/converter-EEK-XAG>; rel='alternate'; hreflang='tr-BG'\n" - "Link: <http://www.forexticket-cy.com/tr/currency/converter-EEK-XAG>; rel='alternate'; hreflang='tr-CY'\n" - "Link: <http://www.forexticket.ir/fa/currency/converter-EEK-XAG>; rel='alternate'; hreflang='fa-IR'\n" - "Link: <http://www.forexticket.af/fa/currency/converter-EEK-XAG>; rel='alternate'; hreflang='fa-AF'\n" - "Link: <http://www.forexticket.cd/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-CD'\n" - "Link: <http://www.forexticket.fr/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-FR'\n" - "Link: <http://www.forexticket.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-CA'\n" - "Link: <http://www.forexticket.mg/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-MG'\n" - "Link: <http://www.forexticket.cm/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-CM'\n" - "Link: <http://www.forexticket-kh.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-KH'\n" - "Link: <http://www.forexticket-ml.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-ML'\n" - "Link: <http://www.forexticket-sn.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-SN'\n" - "Link: <http://www.forexticket-tn.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-TN'\n" - "Link: <http://www.forexticket-td.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-TD'\n" - "Link: <http://www.forexticket.be/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-BE'\n" - "Link: <http://www.forexticket-gn.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-GN'\n" - "Link: <http://www.forexticket.ht/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-HT'\n" - "Link: <http://www.forexticket.ch/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-CH'\n" - "Link: <http://www.forexticket.la/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-LA'\n" - "Link: <http://www.forexticket.lu/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-LU'\n" - "Link: <http://www.forexticket-th.com/th/currency/converter-EEK-XAG>; rel='alternate'; hreflang='th-TH'\n" - "Link: <http://www.forexticket.co.uk/cy/currency/converter-EEK-XAG>; rel='alternate'; hreflang='cy-GB'\n" - "Link: <http://www.forexticket.co.uk/ga/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ga-GB'\n" - "Link: <http://www.forexticket.it/it/convertitore/valuta-EEK-XAG>; rel='alternate'; hreflang='it-IT'\n" - "Link: <http://www.forexticket.ch/it/convertitore/valuta-EEK-XAG>; rel='alternate'; hreflang='it-CH'\n" - "Link: <http://www.forexticket.co.za/af/currency/converter-EEK-XAG>; rel='alternate'; hreflang='af-ZA'\n" - "Link: <http://www.forexticket.kr/ko/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ko-KR'\n" - "Link: <http://www.forexticket-ua.com/uk/currency/converter-EEK-XAG>; rel='alternate'; hreflang='uk-UA'\n" - "Link: <http://www.forexticket-tz.com/sw/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sw-TZ'\n" - "Link: <http://www.forexticket.co.ke/sw/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sw-KE'\n" - "Link: <http://www.forexticket.pl/pl/currency/converter-EEK-XAG>; rel='alternate'; hreflang='pl-PL'\n" - "Link: <http://www.forexticket.com.my/ms/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ms-MY'\n" - "Link: <http://www.forexticket.sg/ms/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ms-SG'\n" - "Link: <http://www.forexticket.ro/ro/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ro-RO'\n" - "Link: <http://www.forexticket.nl/nl/currency/converter-EEK-XAG>; rel='alternate'; hreflang='nl-NL'\n" - "Link: <http://www.forexticket.be/nl/currency/converter-EEK-XAG>; rel='alternate'; hreflang='nl-BE'\n" - "Link: <http://www.forexticket.gr/el/currency/converter-EEK-XAG>; rel='alternate'; hreflang='el-GR'\n" - "Link: <http://www.forexticket-al.com/el/currency/converter-EEK-XAG>; rel='alternate'; hreflang='el-AL'\n" - "Link: <http://www.forexticket-cy.com/el/currency/converter-EEK-XAG>; rel='alternate'; hreflang='el-CY'\n" - "Link: <http://www.forexticket.cz/cs/currency/converter-EEK-XAG>; rel='alternate'; hreflang='cs-CZ'\n" - "Link: <http://www.forexticket.hu/hu/currency/converter-EEK-XAG>; rel='alternate'; hreflang='hu-HU'\n" - "Link: <http://www.forexticket.se/sv/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sv-SE'\n" - "Link: <http://www.forexticket.eu/sv/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sv-FI'\n" - "Link: <http://www.forexticket.co.il/iw/currency/converter-EEK-XAG>; rel='alternate'; hreflang='iw-IL'\n" - "Link: <http://www.forexticket.co.il/yi/currency/converter-EEK-XAG>; rel='alternate'; hreflang='yi-IL'\n" - "Link: <http://www.forexticket-bg.com/bg/currency/converter-EEK-XAG>; rel='alternate'; hreflang='bg-BG'\n" - "Link: <http://www.forexticket.es/ca/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ca-ES'\n" - "Link: <http://www.forexticket.es/gl/currency/converter-EEK-XAG>; rel='alternate'; hreflang='gl-ES'\n" - "Link: <http://www.forexticket.dk/da/currency/converter-EEK-XAG>; rel='alternate'; hreflang='da-DK'\n" - "Link: <http://www.forexticket.eu/fi/currency/converter-EEK-XAG>; rel='alternate'; hreflang='fi-FI'\n" - "Link: <http://www.forexticket-hr.com/hr/currency/converter-EEK-XAG>; rel='alternate'; hreflang='hr-HR'\n" - "Link: <http://www.forexticket-hr.com/sr/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sr-HR'\n" - "Link: <http://www.forexticket.me/sr/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sr-ME'\n" - "Link: <http://www.forexticket.lt/lt/currency/converter-EEK-XAG>; rel='alternate'; hreflang='lt-LT'\n" - "Link: <http://www.forexticket-al.com/sq/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sq-AL'\n" - "Link: <http://www.forexticket.lv/lv/currency/converter-EEK-XAG>; rel='alternate'; hreflang='lv-LV'\n" - "Link: <http://www.forexticket.co.ee/et/currency/converter-EEK-XAG>; rel='alternate'; hreflang='et-EE'\n" - "Vary: Accept-Encoding,User-Agent\n" - "Content-Encoding: gzip\n" - "Keep-Alive: timeout=1, max=100\n" - "Connection: Keep-Alive\n" - "Transfer-Encoding: chunked\n" - "Content-Type: text/html\n" - "\n"; + "Date: Thu, 15 Nov 2012 22:38:28 GMT\n" + "Server: Apache/2\n" + "X-Powered-By: PHP/5.2.17\n" + "Set-Cookie: PHPSESSID=6d69474d1cc019d7d82714c9472bc6d6; path=/\n" + "Expires: Thu, 19 Nov 1981 08:52:00 GMT\n" + "Cache-Control: no-store, no-cache, must-revalidate, post-check=0, pre-check=0\n" + "Pragma: no-cache\n" + "Link: <http://www.forexticket.cn.com/zh/currency/converter-EEK-XAG>; rel='alternate'; hreflang='zh-CN'\n" + "Link: <http://www.forexticket.tw/zh/currency/converter-EEK-XAG>; rel='alternate'; hreflang='zh-TW'\n" + "Link: <http://www.forexticket.hk/zh/currency/converter-EEK-XAG>; rel='alternate'; hreflang='zh-HK'\n" + "Link: <http://www.forexticket.sg/zh/currency/converter-EEK-XAG>; rel='alternate'; hreflang='zh-SG'\n" + "Link: <http://www.forexticket.in/hi/currency/converter-EEK-XAG>; rel='alternate'; hreflang='hi-IN'\n" + "Link: <http://www.forexticket.com.fj/hi/currency/converter-EEK-XAG>; rel='alternate'; hreflang='hi-FJ'\n" + "Link: <http://www.forexticket.in/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-IN'\n" + "Link: <http://www.forexticket.us/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-US'\n" + "Link: <http://www.forexticket.com.pk/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-PK'\n" + "Link: <http://www.forexticket-bd.com/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-BD'\n" + "Link: <http://www.forexticket-ng.com/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-NG'\n" + "Link: <http://www.forexticket.co.uk/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-GB'\n" + "Link: <http://www.forexticket.co.za/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-ZA'\n" + "Link: <http://www.forexticket.co.ke/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-KE'\n" + "Link: <http://www.forexticket.com/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-CA'\n" + "Link: <http://www.forexticket-gh.com/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-GH'\n" + "Link: <http://www.forexticket.biz/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-AU'\n" + "Link: <http://www.forexticket.cm/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-CM'\n" + "Link: <http://www.forexticket-kh.com/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-KH'\n" + "Link: <http://www.forexticket.hk/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-HK'\n" + "Link: <http://www.forexticket.la/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-LA'\n" + "Link: <http://www.forexticket.sg/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-SG'\n" + "Link: <http://www.forexticket.co.nz/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-NZ'\n" + "Link: <http://www.forexticket.com.pr/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-PR'\n" + "Link: <http://www.forexticket.com.fj/en/currency/converter-EEK-XAG>; rel='alternate'; hreflang='en-FJ'\n" + "Link: <http://www.forexticket.us/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-US'\n" + "Link: <http://www.forexticket.mx/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-MX'\n" + "Link: <http://www.forexticket.co/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-CO'\n" + "Link: <http://www.forexticket.com.ar/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-AR'\n" + "Link: <http://www.forexticket-pe.com/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-PE'\n" + "Link: <http://www.forexticket.co.ve/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-VE'\n" + "Link: <http://www.forexticket.cl/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-CL'\n" + "Link: <http://www.forexticket.ec/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-EC'\n" + "Link: <http://www.forexticket.com.gt/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-GT'\n" + "Link: <http://www.forexticket.bo/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-BO'\n" + "Link: <http://www.forexticket.hn/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-HN'\n" + "Link: <http://www.forexticket.com.py/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-PY'\n" + "Link: <http://www.forexticket.es/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-ES'\n" + "Link: <http://www.forexticket.com.sv/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-SV'\n" + "Link: <http://www.forexticket.com.ni/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-NI'\n" + "Link: <http://www.forexticket.co.cr/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-CR'\n" + "Link: <http://www.forexticket.com.pr/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-PR'\n" + "Link: <http://www.forexticket.com.uy/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-UY'\n" + "Link: <http://www.forexticket.com.pa/es/cambio/divisas-EEK-XAG>; rel='alternate'; hreflang='es-PA'\n" + "Link: <http://www.forexticket.asia.com/id/currency/converter-EEK-XAG>; rel='alternate'; hreflang='id-ID'\n" + "Link: <http://www.forexticket.com.br/pt/moeda/conversor-EEK-XAG>; rel='alternate'; hreflang='pt-BR'\n" + "Link: <http://www.forexticket-mz.com/pt/moeda/conversor-EEK-XAG>; rel='alternate'; hreflang='pt-MZ'\n" + "Link: <http://www.forexticket.com.pt/pt/moeda/conversor-EEK-XAG>; rel='alternate'; hreflang='pt-PT'\n" + "Link: <http://www.forexticket.tl/pt/moeda/conversor-EEK-XAG>; rel='alternate'; hreflang='pt-TL'\n" + "Link: <http://www.forexticket.ru/ru/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ru-RU'\n" + "Link: <http://www.forexticket-kz.com/ru/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ru-KZ'\n" + "Link: <http://www.forexticket-tj.com/ru/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ru-TJ'\n" + "Link: <http://www.forexticket-kg.com/ru/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ru-KG'\n" + "Link: <http://www.forexticket-ge.com/ru/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ru-GE'\n" + "Link: <http://www.forexticket.mn/ru/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ru-MN'\n" + "Link: <http://www.forexticket.jp/ja/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ja-JP'\n" + "Link: <http://www.forexticket-ph.com/tl/currency/converter-EEK-XAG>; rel='alternate'; hreflang='tl-PH'\n" + "Link: <http://www.forexticket.vn/vi/currency/converter-EEK-XAG>; rel='alternate'; hreflang='vi-VN'\n" + "Link: <http://www.forexticket.de/de/waehrungsumrechner/devisen-EEK-XAG>; rel='alternate'; hreflang='de-DE'\n" + "Link: <http://www.forexticket.be/de/waehrungsumrechner/devisen-EEK-XAG>; rel='alternate'; hreflang='de-BE'\n" + "Link: <http://www.forexticket.at/de/waehrungsumrechner/devisen-EEK-XAG>; rel='alternate'; hreflang='de-AT'\n" + "Link: <http://www.forexticket.ch/de/waehrungsumrechner/devisen-EEK-XAG>; rel='alternate'; hreflang='de-CH'\n" + "Link: <http://www.forexticket.lu/de/waehrungsumrechner/devisen-EEK-XAG>; rel='alternate'; hreflang='de-LU'\n" + "Link: <http://www.forexticket.li/de/waehrungsumrechner/devisen-EEK-XAG>; rel='alternate'; hreflang='de-LI'\n" + "Link: <http://www.forexticket.de/de/waehrungsumrechner/devisen-EEK-XAG>; rel='canonical'\n" + "Link: <http://www.forexticket-eg.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-EG'\n" + "Link: <http://www.forexticket-dz.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-DZ'\n" + "Link: <http://www.forexticket-ma.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-MA'\n" + "Link: <http://www.forexticket-iq.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-IQ'\n" + "Link: <http://www.forexticket-sa.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-SA'\n" + "Link: <http://www.forexticket-sy.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-SY'\n" + "Link: <http://www.forexticket-tn.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-TN'\n" + "Link: <http://www.forexticket-td.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-TD'\n" + "Link: <http://www.forexticket-so.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-SO'\n" + "Link: <http://www.forexticket.co.il/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-IL'\n" + "Link: <http://www.forexticket-jo.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-JO'\n" + "Link: <http://www.forexticket.ae/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-AE'\n" + "Link: <http://www.forexticket-lb.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-LB'\n" + "Link: <http://www.forexticket-om.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-OM'\n" + "Link: <http://www.forexticket-kw.com/ar/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ar-KW'\n" + "Link: <http://www.forexticket-tr.com/tr/currency/converter-EEK-XAG>; rel='alternate'; hreflang='tr-TR'\n" + "Link: <http://www.forexticket-bg.com/tr/currency/converter-EEK-XAG>; rel='alternate'; hreflang='tr-BG'\n" + "Link: <http://www.forexticket-cy.com/tr/currency/converter-EEK-XAG>; rel='alternate'; hreflang='tr-CY'\n" + "Link: <http://www.forexticket.ir/fa/currency/converter-EEK-XAG>; rel='alternate'; hreflang='fa-IR'\n" + "Link: <http://www.forexticket.af/fa/currency/converter-EEK-XAG>; rel='alternate'; hreflang='fa-AF'\n" + "Link: <http://www.forexticket.cd/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-CD'\n" + "Link: <http://www.forexticket.fr/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-FR'\n" + "Link: <http://www.forexticket.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-CA'\n" + "Link: <http://www.forexticket.mg/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-MG'\n" + "Link: <http://www.forexticket.cm/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-CM'\n" + "Link: <http://www.forexticket-kh.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-KH'\n" + "Link: <http://www.forexticket-ml.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-ML'\n" + "Link: <http://www.forexticket-sn.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-SN'\n" + "Link: <http://www.forexticket-tn.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-TN'\n" + "Link: <http://www.forexticket-td.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-TD'\n" + "Link: <http://www.forexticket.be/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-BE'\n" + "Link: <http://www.forexticket-gn.com/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-GN'\n" + "Link: <http://www.forexticket.ht/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-HT'\n" + "Link: <http://www.forexticket.ch/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-CH'\n" + "Link: <http://www.forexticket.la/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-LA'\n" + "Link: <http://www.forexticket.lu/fr/conversion/monnaie-EEK-XAG>; rel='alternate'; hreflang='fr-LU'\n" + "Link: <http://www.forexticket-th.com/th/currency/converter-EEK-XAG>; rel='alternate'; hreflang='th-TH'\n" + "Link: <http://www.forexticket.co.uk/cy/currency/converter-EEK-XAG>; rel='alternate'; hreflang='cy-GB'\n" + "Link: <http://www.forexticket.co.uk/ga/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ga-GB'\n" + "Link: <http://www.forexticket.it/it/convertitore/valuta-EEK-XAG>; rel='alternate'; hreflang='it-IT'\n" + "Link: <http://www.forexticket.ch/it/convertitore/valuta-EEK-XAG>; rel='alternate'; hreflang='it-CH'\n" + "Link: <http://www.forexticket.co.za/af/currency/converter-EEK-XAG>; rel='alternate'; hreflang='af-ZA'\n" + "Link: <http://www.forexticket.kr/ko/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ko-KR'\n" + "Link: <http://www.forexticket-ua.com/uk/currency/converter-EEK-XAG>; rel='alternate'; hreflang='uk-UA'\n" + "Link: <http://www.forexticket-tz.com/sw/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sw-TZ'\n" + "Link: <http://www.forexticket.co.ke/sw/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sw-KE'\n" + "Link: <http://www.forexticket.pl/pl/currency/converter-EEK-XAG>; rel='alternate'; hreflang='pl-PL'\n" + "Link: <http://www.forexticket.com.my/ms/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ms-MY'\n" + "Link: <http://www.forexticket.sg/ms/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ms-SG'\n" + "Link: <http://www.forexticket.ro/ro/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ro-RO'\n" + "Link: <http://www.forexticket.nl/nl/currency/converter-EEK-XAG>; rel='alternate'; hreflang='nl-NL'\n" + "Link: <http://www.forexticket.be/nl/currency/converter-EEK-XAG>; rel='alternate'; hreflang='nl-BE'\n" + "Link: <http://www.forexticket.gr/el/currency/converter-EEK-XAG>; rel='alternate'; hreflang='el-GR'\n" + "Link: <http://www.forexticket-al.com/el/currency/converter-EEK-XAG>; rel='alternate'; hreflang='el-AL'\n" + "Link: <http://www.forexticket-cy.com/el/currency/converter-EEK-XAG>; rel='alternate'; hreflang='el-CY'\n" + "Link: <http://www.forexticket.cz/cs/currency/converter-EEK-XAG>; rel='alternate'; hreflang='cs-CZ'\n" + "Link: <http://www.forexticket.hu/hu/currency/converter-EEK-XAG>; rel='alternate'; hreflang='hu-HU'\n" + "Link: <http://www.forexticket.se/sv/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sv-SE'\n" + "Link: <http://www.forexticket.eu/sv/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sv-FI'\n" + "Link: <http://www.forexticket.co.il/iw/currency/converter-EEK-XAG>; rel='alternate'; hreflang='iw-IL'\n" + "Link: <http://www.forexticket.co.il/yi/currency/converter-EEK-XAG>; rel='alternate'; hreflang='yi-IL'\n" + "Link: <http://www.forexticket-bg.com/bg/currency/converter-EEK-XAG>; rel='alternate'; hreflang='bg-BG'\n" + "Link: <http://www.forexticket.es/ca/currency/converter-EEK-XAG>; rel='alternate'; hreflang='ca-ES'\n" + "Link: <http://www.forexticket.es/gl/currency/converter-EEK-XAG>; rel='alternate'; hreflang='gl-ES'\n" + "Link: <http://www.forexticket.dk/da/currency/converter-EEK-XAG>; rel='alternate'; hreflang='da-DK'\n" + "Link: <http://www.forexticket.eu/fi/currency/converter-EEK-XAG>; rel='alternate'; hreflang='fi-FI'\n" + "Link: <http://www.forexticket-hr.com/hr/currency/converter-EEK-XAG>; rel='alternate'; hreflang='hr-HR'\n" + "Link: <http://www.forexticket-hr.com/sr/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sr-HR'\n" + "Link: <http://www.forexticket.me/sr/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sr-ME'\n" + "Link: <http://www.forexticket.lt/lt/currency/converter-EEK-XAG>; rel='alternate'; hreflang='lt-LT'\n" + "Link: <http://www.forexticket-al.com/sq/currency/converter-EEK-XAG>; rel='alternate'; hreflang='sq-AL'\n" + "Link: <http://www.forexticket.lv/lv/currency/converter-EEK-XAG>; rel='alternate'; hreflang='lv-LV'\n" + "Link: <http://www.forexticket.co.ee/et/currency/converter-EEK-XAG>; rel='alternate'; hreflang='et-EE'\n" + "Vary: Accept-Encoding,User-Agent\n" + "Content-Encoding: gzip\n" + "Keep-Alive: timeout=1, max=100\n" + "Connection: Keep-Alive\n" + "Transfer-Encoding: chunked\n" + "Content-Type: text/html\n" + "\n"; diff --git a/library/cpp/http/fetch/ya.make b/library/cpp/http/fetch/ya.make index 7737127463..9962b82ee1 100644 --- a/library/cpp/http/fetch/ya.make +++ b/library/cpp/http/fetch/ya.make @@ -1,19 +1,19 @@ -LIBRARY() +LIBRARY() OWNER( g:zora ) -PEERDIR( - contrib/libs/zlib +PEERDIR( + contrib/libs/zlib library/cpp/charset library/cpp/digest/md5 library/cpp/http/misc library/cpp/logger library/cpp/mime/types library/cpp/uri -) - +) + SRCS( http_digest.cpp http_socket.cpp @@ -32,7 +32,7 @@ SRCS( GENERATE_ENUM_SERIALIZATION(httpheader.h) SET(RAGEL6_FLAGS -CF1) - + END() RECURSE_FOR_TESTS(ut) diff --git a/library/cpp/http/io/chunk.cpp b/library/cpp/http/io/chunk.cpp index 6975d9eac1..80a423e7c9 100644 --- a/library/cpp/http/io/chunk.cpp +++ b/library/cpp/http/io/chunk.cpp @@ -1,70 +1,70 @@ -#include "chunk.h" - +#include "chunk.h" + #include "headers.h" -#include <util/string/cast.h> -#include <util/generic/utility.h> -#include <util/generic/yexception.h> - +#include <util/string/cast.h> +#include <util/generic/utility.h> +#include <util/generic/yexception.h> + static inline size_t ParseHex(const TString& s) { - if (s.empty()) { + if (s.empty()) { ythrow yexception() << "can not parse chunk length(empty string)"; - } - - size_t ret = 0; - + } + + size_t ret = 0; + for (TString::const_iterator c = s.begin(); c != s.end(); ++c) { - const char ch = *c; - - if (ch >= '0' && ch <= '9') { - ret *= 16; - ret += ch - '0'; - } else if (ch >= 'a' && ch <= 'f') { - ret *= 16; - ret += 10 + ch - 'a'; - } else if (ch >= 'A' && ch <= 'F') { - ret *= 16; - ret += 10 + ch - 'A'; - } else if (ch == ';') { - break; - } else if (isspace(ch)) { - continue; - } else { + const char ch = *c; + + if (ch >= '0' && ch <= '9') { + ret *= 16; + ret += ch - '0'; + } else if (ch >= 'a' && ch <= 'f') { + ret *= 16; + ret += 10 + ch - 'a'; + } else if (ch >= 'A' && ch <= 'F') { + ret *= 16; + ret += 10 + ch - 'A'; + } else if (ch == ';') { + break; + } else if (isspace(ch)) { + continue; + } else { ythrow yexception() << "can not parse chunk length(" << s.data() << ")"; - } - } - - return ret; -} - -static inline char* ToHex(size_t len, char* buf) { - do { - const size_t val = len % 16; - - *--buf = (val < 10) ? (val + '0') : (val - 10 + 'a'); - len /= 16; - } while (len); - - return buf; -} - -class TChunkedInput::TImpl { -public: + } + } + + return ret; +} + +static inline char* ToHex(size_t len, char* buf) { + do { + const size_t val = len % 16; + + *--buf = (val < 10) ? (val + '0') : (val - 10 + 'a'); + len /= 16; + } while (len); + + return buf; +} + +class TChunkedInput::TImpl { +public: inline TImpl(IInputStream* slave, TMaybe<THttpHeaders>* trailers) - : Slave_(slave) + : Slave_(slave) , Trailers_(trailers) - , Pending_(0) - , LastChunkReaded_(false) - { + , Pending_(0) + , LastChunkReaded_(false) + { if (Trailers_) { Trailers_->Clear(); } - } - + } + inline ~TImpl() { - } - - inline size_t Read(void* buf, size_t len) { + } + + inline size_t Read(void* buf, size_t len) { return Perform(len, [this, buf](size_t toRead) { return Slave_->Read(buf, toRead); }); } @@ -73,174 +73,174 @@ public: } private: - template <class Operation> + template <class Operation> inline size_t Perform(size_t len, const Operation& operation) { - if (!HavePendingData()) { - return 0; - } - + if (!HavePendingData()) { + return 0; + } + const size_t toProcess = Min(Pending_, len); - + if (toProcess) { const size_t processed = operation(toProcess); - + if (!processed) { - ythrow yexception() << "malformed http chunk"; - } - + ythrow yexception() << "malformed http chunk"; + } + Pending_ -= processed; - + return processed; - } - - return 0; - } - - inline bool HavePendingData() { - if (LastChunkReaded_) { - return false; - } - - if (!Pending_) { - if (!ProceedToNextChunk()) { - return false; - } - } - - return true; - } - - inline bool ProceedToNextChunk() { + } + + return 0; + } + + inline bool HavePendingData() { + if (LastChunkReaded_) { + return false; + } + + if (!Pending_) { + if (!ProceedToNextChunk()) { + return false; + } + } + + return true; + } + + inline bool ProceedToNextChunk() { TString len(Slave_->ReadLine()); - - if (len.empty()) { - /* - * skip crlf from previous chunk - */ - - len = Slave_->ReadLine(); - } - - Pending_ = ParseHex(len); - - if (Pending_) { - return true; - } - + + if (len.empty()) { + /* + * skip crlf from previous chunk + */ + + len = Slave_->ReadLine(); + } + + Pending_ = ParseHex(len); + + if (Pending_) { + return true; + } + if (Trailers_) { Trailers_->ConstructInPlace(Slave_); } - LastChunkReaded_ = true; - - return false; - } - -private: + LastChunkReaded_ = true; + + return false; + } + +private: IInputStream* Slave_; TMaybe<THttpHeaders>* Trailers_; - size_t Pending_; - bool LastChunkReaded_; -}; - + size_t Pending_; + bool LastChunkReaded_; +}; + TChunkedInput::TChunkedInput(IInputStream* slave, TMaybe<THttpHeaders>* trailers) : Impl_(new TImpl(slave, trailers)) -{ -} - +{ +} + TChunkedInput::~TChunkedInput() { -} - -size_t TChunkedInput::DoRead(void* buf, size_t len) { - return Impl_->Read(buf, len); -} - +} + +size_t TChunkedInput::DoRead(void* buf, size_t len) { + return Impl_->Read(buf, len); +} + size_t TChunkedInput::DoSkip(size_t len) { return Impl_->Skip(len); } -class TChunkedOutput::TImpl { +class TChunkedOutput::TImpl { typedef IOutputStream::TPart TPart; - -public: + +public: inline TImpl(IOutputStream* slave) - : Slave_(slave) - { - } - + : Slave_(slave) + { + } + inline ~TImpl() { - } - - inline void Write(const void* buf, size_t len) { - const char* ptr = (const char*)buf; - - while (len) { - const size_t portion = Min<size_t>(len, 1024 * 16); - - WriteImpl(ptr, portion); - - ptr += portion; - len -= portion; - } - } - - inline void WriteImpl(const void* buf, size_t len) { - char tmp[32]; - char* e = tmp + sizeof(tmp); - char* b = ToHex(len, e); - - const TPart parts[] = { - TPart(b, e - b), - TPart::CrLf(), - TPart(buf, len), - TPart::CrLf(), - }; - - Slave_->Write(parts, sizeof(parts) / sizeof(*parts)); - } - - inline void Flush() { - Slave_->Flush(); - } - - inline void Finish() { - Slave_->Write("0\r\n\r\n", 5); - - Flush(); - } - -private: + } + + inline void Write(const void* buf, size_t len) { + const char* ptr = (const char*)buf; + + while (len) { + const size_t portion = Min<size_t>(len, 1024 * 16); + + WriteImpl(ptr, portion); + + ptr += portion; + len -= portion; + } + } + + inline void WriteImpl(const void* buf, size_t len) { + char tmp[32]; + char* e = tmp + sizeof(tmp); + char* b = ToHex(len, e); + + const TPart parts[] = { + TPart(b, e - b), + TPart::CrLf(), + TPart(buf, len), + TPart::CrLf(), + }; + + Slave_->Write(parts, sizeof(parts) / sizeof(*parts)); + } + + inline void Flush() { + Slave_->Flush(); + } + + inline void Finish() { + Slave_->Write("0\r\n\r\n", 5); + + Flush(); + } + +private: IOutputStream* Slave_; -}; - +}; + TChunkedOutput::TChunkedOutput(IOutputStream* slave) - : Impl_(new TImpl(slave)) -{ -} - + : Impl_(new TImpl(slave)) +{ +} + TChunkedOutput::~TChunkedOutput() { - try { - Finish(); - } catch (...) { - } -} - -void TChunkedOutput::DoWrite(const void* buf, size_t len) { - if (Impl_.Get()) { - Impl_->Write(buf, len); - } else { + try { + Finish(); + } catch (...) { + } +} + +void TChunkedOutput::DoWrite(const void* buf, size_t len) { + if (Impl_.Get()) { + Impl_->Write(buf, len); + } else { ythrow yexception() << "can not write to finished stream"; - } -} - -void TChunkedOutput::DoFlush() { - if (Impl_.Get()) { - Impl_->Flush(); - } -} - -void TChunkedOutput::DoFinish() { - if (Impl_.Get()) { - Impl_->Finish(); - Impl_.Destroy(); - } -} + } +} + +void TChunkedOutput::DoFlush() { + if (Impl_.Get()) { + Impl_->Flush(); + } +} + +void TChunkedOutput::DoFinish() { + if (Impl_.Get()) { + Impl_->Finish(); + Impl_.Destroy(); + } +} diff --git a/library/cpp/http/io/chunk.h b/library/cpp/http/io/chunk.h index 88d89fafda..a77f1b1a9d 100644 --- a/library/cpp/http/io/chunk.h +++ b/library/cpp/http/io/chunk.h @@ -1,9 +1,9 @@ #pragma once - + #include <util/stream/output.h> #include <util/generic/maybe.h> #include <util/generic/ptr.h> - + class THttpHeaders; /// @addtogroup Streams_Chunked @@ -12,36 +12,36 @@ class THttpHeaders; /// @details Последовательное чтение блоков данных. Предполагается, что /// данные записаны в виде <длина блока><блок данных>. class TChunkedInput: public IInputStream { -public: +public: /// Если передан указатель на trailers, то туда будут записаны HTTP trailer'ы (возможно пустые), /// которые идут после чанков. TChunkedInput(IInputStream* slave, TMaybe<THttpHeaders>* trailers = nullptr); ~TChunkedInput() override; - -private: + +private: size_t DoRead(void* buf, size_t len) override; size_t DoSkip(size_t len) override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /// Вывод данных порциями. /// @details Вывод данных блоками в виде <длина блока><блок данных>. Если объем /// данных превышает 64K, они записываются в виде n блоков по 64K + то, что осталось. class TChunkedOutput: public IOutputStream { -public: +public: TChunkedOutput(IOutputStream* slave); ~TChunkedOutput() override; - -private: + +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; + +private: + class TImpl; + THolder<TImpl> Impl_; +}; /// @} diff --git a/library/cpp/http/io/chunk_ut.cpp b/library/cpp/http/io/chunk_ut.cpp index da283f8568..82f9c90b74 100644 --- a/library/cpp/http/io/chunk_ut.cpp +++ b/library/cpp/http/io/chunk_ut.cpp @@ -1,16 +1,16 @@ -#include "chunk.h" - +#include "chunk.h" + #include <library/cpp/testing/unittest/registar.h> - + #include <util/stream/file.h> -#include <util/system/tempfile.h> +#include <util/system/tempfile.h> #include <util/stream/null.h> - -#define CDATA "./chunkedio" - + +#define CDATA "./chunkedio" + Y_UNIT_TEST_SUITE(TestChunkedIO) { - static const char test_data[] = "87s6cfbsudg cuisg s igasidftasiy tfrcua6s"; - + static const char test_data[] = "87s6cfbsudg cuisg s igasidftasiy tfrcua6s"; + TString CombString(const TString& s, size_t chunkSize) { TString result; for (size_t pos = 0; pos < s.size(); pos += 2 * chunkSize) @@ -58,48 +58,48 @@ Y_UNIT_TEST_SUITE(TestChunkedIO) { } Y_UNIT_TEST(TestChunkedIo) { - TTempFile tmpFile(CDATA); + TTempFile tmpFile(CDATA); TString tmp; - - { + + { TUnbufferedFileOutput fo(CDATA); - TChunkedOutput co(&fo); + TChunkedOutput co(&fo); WriteTestData(&co, &tmp); } - + { TUnbufferedFileInput fi(CDATA); TChunkedInput ci(&fi); TString r; - + ReadInSmallChunks(&ci, &r); UNIT_ASSERT_EQUAL(r, tmp); - } - - { + } + + { TUnbufferedFileInput fi(CDATA); - TChunkedInput ci(&fi); + TChunkedInput ci(&fi); TString r; - + ReadCombed(&ci, &r, 11); - + UNIT_ASSERT_EQUAL(r, CombString(tmp, 11)); - } - } - + } + } + Y_UNIT_TEST(TestBadChunk) { - bool hasError = false; - - try { + bool hasError = false; + + try { TString badChunk = "10\r\nqwerty"; TMemoryInput mi(badChunk.data(), badChunk.size()); - TChunkedInput ci(&mi); - TransferData(&ci, &Cnull); - } catch (...) { - hasError = true; - } - - UNIT_ASSERT(hasError); - } -} + TChunkedInput ci(&mi); + TransferData(&ci, &Cnull); + } catch (...) { + hasError = true; + } + + UNIT_ASSERT(hasError); + } +} diff --git a/library/cpp/http/io/compression.cpp b/library/cpp/http/io/compression.cpp index 8fa1f62ae6..0c922ad566 100644 --- a/library/cpp/http/io/compression.cpp +++ b/library/cpp/http/io/compression.cpp @@ -1,66 +1,66 @@ #include "compression.h" - -#if defined(ENABLE_GPL) + +#if defined(ENABLE_GPL) #include <library/cpp/streams/lz/lz.h> -#endif - +#endif + #include <library/cpp/streams/brotli/brotli.h> #include <library/cpp/streams/lzma/lzma.h> #include <library/cpp/streams/bzip2/bzip2.h> - + #include <library/cpp/blockcodecs/stream.h> #include <library/cpp/blockcodecs/codecs.h> - -#include <util/stream/zlib.h> - - + +#include <util/stream/zlib.h> + + TCompressionCodecFactory::TCompressionCodecFactory() { auto gzip = [](auto s) { return MakeHolder<TZLibDecompress>(s); - }; - + }; + Add("gzip", gzip, [](auto s) { return MakeHolder<TZLibCompress>(s, ZLib::GZip); }); Add("deflate", gzip, [](auto s) { return MakeHolder<TZLibCompress>(s, ZLib::ZLib); }); Add("br", [](auto s) { return MakeHolder<TBrotliDecompress>(s); }, [](auto s) { return MakeHolder<TBrotliCompress>(s, 4); }); Add("x-gzip", gzip, [](auto s) { return MakeHolder<TZLibCompress>(s, ZLib::GZip); }); Add("x-deflate", gzip, [](auto s) { return MakeHolder<TZLibCompress>(s, ZLib::ZLib); }); - -#if defined(ENABLE_GPL) + +#if defined(ENABLE_GPL) const ui16 bs = 32 * 1024; - + Add("y-lzo", [](auto s) { return MakeHolder<TLzoDecompress>(s); }, [bs](auto s) { return MakeHolder<TLazy<TLzoCompress> >(s, bs); }); Add("y-lzf", [](auto s) { return MakeHolder<TLzfDecompress>(s); }, [bs](auto s) { return MakeHolder<TLazy<TLzfCompress> >(s, bs); }); Add("y-lzq", [](auto s) { return MakeHolder<TLzqDecompress>(s); }, [bs](auto s) { return MakeHolder<TLazy<TLzqCompress> >(s, bs); }); -#endif - +#endif + Add("y-bzip2", [](auto s) { return MakeHolder<TBZipDecompress>(s); }, [](auto s) { return MakeHolder<TBZipCompress>(s); }); Add("y-lzma", [](auto s) { return MakeHolder<TLzmaDecompress>(s); }, [](auto s) { return MakeHolder<TLzmaCompress>(s); }); - + for (auto codecName : NBlockCodecs::ListAllCodecs()) { if (codecName.StartsWith("zstd06")) { continue; - } - + } + if (codecName.StartsWith("zstd08")) { continue; - } - + } + auto codec = NBlockCodecs::Codec(codecName); - + auto enc = [codec](auto s) { return MakeHolder<NBlockCodecs::TCodedOutput>(s, codec, 32 * 1024); }; - + auto dec = [codec](auto s) { return MakeHolder<NBlockCodecs::TDecodedInput>(s, codec); - }; - + }; + Add(TString("z-") + codecName, dec, enc); - } -} - + } +} + void TCompressionCodecFactory::Add(TStringBuf name, TDecoderConstructor d, TEncoderConstructor e) { Strings_.emplace_back(name); Codecs_[Strings_.back()] = TCodec{d, e}; BestCodecs_.emplace_back(Strings_.back()); -} +} diff --git a/library/cpp/http/io/compression.h b/library/cpp/http/io/compression.h index f16c4a18eb..b566bbb796 100644 --- a/library/cpp/http/io/compression.h +++ b/library/cpp/http/io/compression.h @@ -1,17 +1,17 @@ #pragma once - + #include "stream.h" - + #include <util/generic/deque.h> #include <util/generic/hash.h> - + class TCompressionCodecFactory { -public: +public: using TDecoderConstructor = std::function<THolder<IInputStream>(IInputStream*)>; using TEncoderConstructor = std::function<THolder<IOutputStream>(IOutputStream*)>; - + TCompressionCodecFactory(); - + static inline TCompressionCodecFactory& Instance() noexcept { return *SingletonWithPriority<TCompressionCodecFactory, 0>(); } @@ -20,10 +20,10 @@ public: if (auto codec = Codecs_.FindPtr(name)) { return &codec->Decoder; } - + return nullptr; } - + inline const TEncoderConstructor* FindEncoder(TStringBuf name) const { if (auto codec = Codecs_.FindPtr(name)) { return &codec->Encoder; @@ -31,23 +31,23 @@ public: return nullptr; } - + inline TArrayRef<const TStringBuf> GetBestCodecs() const { return BestCodecs_; } -private: +private: void Add(TStringBuf name, TDecoderConstructor d, TEncoderConstructor e); - + struct TCodec { TDecoderConstructor Decoder; TEncoderConstructor Encoder; }; - + TDeque<TString> Strings_; THashMap<TStringBuf, TCodec> Codecs_; TVector<TStringBuf> BestCodecs_; -}; +}; namespace NHttp { template <typename F> diff --git a/library/cpp/http/io/compression_ut.cpp b/library/cpp/http/io/compression_ut.cpp index 2f3d131f8c..04a3f85689 100644 --- a/library/cpp/http/io/compression_ut.cpp +++ b/library/cpp/http/io/compression_ut.cpp @@ -1,34 +1,34 @@ -#include "stream.h" +#include "stream.h" #include "compression.h" - + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - -#include <util/stream/zlib.h> + +#include <util/stream/zlib.h> #include <util/generic/hash_set.h> - + Y_UNIT_TEST_SUITE(THttpCompressionTest) { static const TString DATA = "I'm a teapot"; Y_UNIT_TEST(TestGetBestCodecs) { UNIT_ASSERT(TCompressionCodecFactory::Instance().GetBestCodecs().size() > 0); - } + } Y_UNIT_TEST(TestEncoder) { TStringStream buffer; - { + { auto encoder = TCompressionCodecFactory::Instance().FindEncoder("gzip"); UNIT_ASSERT(encoder); - + auto encodedStream = (*encoder)(&buffer); encodedStream->Write(DATA); - } + } TZLibDecompress decompressor(&buffer); UNIT_ASSERT_EQUAL(decompressor.ReadAll(), DATA); } - + Y_UNIT_TEST(TestDecoder) { TStringStream buffer; diff --git a/library/cpp/http/io/fuzz/main.cpp b/library/cpp/http/io/fuzz/main.cpp index 8ded9c7e32..60bc30848f 100644 --- a/library/cpp/http/io/fuzz/main.cpp +++ b/library/cpp/http/io/fuzz/main.cpp @@ -1,15 +1,15 @@ #include <library/cpp/http/io/stream.h> - -#include <util/generic/vector.h> -#include <util/stream/mem.h> - -extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { - TMemoryInput mi(data, size); - - try { - THttpInput(&mi).ReadAll(); - } catch (...) { - } - - return 0; // Non-zero return values are reserved for future use. -} + +#include <util/generic/vector.h> +#include <util/stream/mem.h> + +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { + TMemoryInput mi(data, size); + + try { + THttpInput(&mi).ReadAll(); + } catch (...) { + } + + return 0; // Non-zero return values are reserved for future use. +} diff --git a/library/cpp/http/io/fuzz/ya.make b/library/cpp/http/io/fuzz/ya.make index 8b3ccb1969..2057830af1 100644 --- a/library/cpp/http/io/fuzz/ya.make +++ b/library/cpp/http/io/fuzz/ya.make @@ -1,18 +1,18 @@ FUZZ() - + OWNER( pg g:util ) - -PEERDIR( + +PEERDIR( library/cpp/http/io -) - +) + SIZE(MEDIUM) -SRCS( - main.cpp -) - -END() +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/http/io/headers.cpp b/library/cpp/http/io/headers.cpp index 4ec27a29e8..5d0d4f895d 100644 --- a/library/cpp/http/io/headers.cpp +++ b/library/cpp/http/io/headers.cpp @@ -1,64 +1,64 @@ -#include "headers.h" -#include "stream.h" - +#include "headers.h" +#include "stream.h" + #include <util/generic/strbuf.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <util/stream/output.h> #include <util/string/ascii.h> -#include <util/string/cast.h> -#include <util/string/strip.h> - +#include <util/string/cast.h> +#include <util/string/strip.h> + static inline TStringBuf Trim(const char* b, const char* e) noexcept { - return StripString(TStringBuf(b, e)); -} - + return StripString(TStringBuf(b, e)); +} + THttpInputHeader::THttpInputHeader(const TStringBuf header) { - size_t pos = header.find(':'); - + size_t pos = header.find(':'); + if (pos == TString::npos) { ythrow THttpParseException() << "can not parse http header(" << TString{header}.Quote() << ")"; - } - + } + Name_ = TString(header.cbegin(), header.cbegin() + pos); Value_ = ::ToString(Trim(header.cbegin() + pos + 1, header.cend())); -} - +} + THttpInputHeader::THttpInputHeader(TString name, TString value) : Name_(std::move(name)) , Value_(std::move(value)) -{ -} - +{ +} + void THttpInputHeader::OutTo(IOutputStream* stream) const { typedef IOutputStream::TPart TPart; - - const TPart parts[] = { - TPart(Name_), - TPart(": ", 2), - TPart(Value_), - TPart::CrLf(), - }; - - stream->Write(parts, sizeof(parts) / sizeof(*parts)); -} - + + const TPart parts[] = { + TPart(Name_), + TPart(": ", 2), + TPart(Value_), + TPart::CrLf(), + }; + + stream->Write(parts, sizeof(parts) / sizeof(*parts)); +} + THttpHeaders::THttpHeaders(IInputStream* stream) { TString header; TString line; - + bool rdOk = stream->ReadLine(header); while (rdOk && !header.empty()) { rdOk = stream->ReadLine(line); - + if (rdOk && ((line[0] == ' ') || (line[0] == '\t'))) { header += line; } else { AddHeader(THttpInputHeader(header)); header = line; - } - } -} - + } + } +} + bool THttpHeaders::HasHeader(const TStringBuf header) const { return FindHeader(header); } @@ -85,24 +85,24 @@ void THttpHeaders::AddOrReplaceHeader(const THttpInputHeader& header) { for (auto& hdr : Headers_) { if (AsciiCompareIgnoreCase(hdr.Name(), header.Name()) == 0) { hdr = header; - return; - } - } - - AddHeader(header); -} - + return; + } + } + + AddHeader(header); +} + void THttpHeaders::AddHeader(THttpInputHeader header) { Headers_.push_back(std::move(header)); -} - +} + void THttpHeaders::OutTo(IOutputStream* stream) const { - for (TConstIterator header = Begin(); header != End(); ++header) { - header->OutTo(stream); - } -} - -template <> + for (TConstIterator header = Begin(); header != End(); ++header) { + header->OutTo(stream); + } +} + +template <> void Out<THttpHeaders>(IOutputStream& out, const THttpHeaders& h) { - h.OutTo(&out); -} + h.OutTo(&out); +} diff --git a/library/cpp/http/io/headers.h b/library/cpp/http/io/headers.h index a71793d1c6..a6e130eaa0 100644 --- a/library/cpp/http/io/headers.h +++ b/library/cpp/http/io/headers.h @@ -1,99 +1,99 @@ #pragma once - + #include <util/generic/string.h> -#include <util/generic/strbuf.h> +#include <util/generic/strbuf.h> #include <util/generic/deque.h> #include <util/generic/vector.h> #include <util/string/cast.h> - + class IInputStream; class IOutputStream; - + /// @addtogroup Streams_HTTP /// @{ /// Объект, содержащий информацию о HTTP-заголовке. class THttpInputHeader { -public: - /// @param[in] header - строка вида 'параметр: значение'. +public: + /// @param[in] header - строка вида 'параметр: значение'. THttpInputHeader(TStringBuf header); - /// @param[in] name - имя параметра. - /// @param[in] value - значение параметра. + /// @param[in] name - имя параметра. + /// @param[in] value - значение параметра. THttpInputHeader(TString name, TString value); - - /// Возвращает имя параметра. + + /// Возвращает имя параметра. inline const TString& Name() const noexcept { - return Name_; - } - - /// Возвращает значение параметра. + return Name_; + } + + /// Возвращает значение параметра. inline const TString& Value() const noexcept { - return Value_; - } - - /// Записывает заголовок вида "имя параметра: значение\r\n" в поток. + return Value_; + } + + /// Записывает заголовок вида "имя параметра: значение\r\n" в поток. void OutTo(IOutputStream* stream) const; - - /// Возвращает строку "имя параметра: значение". + + /// Возвращает строку "имя параметра: значение". inline TString ToString() const { return Name_ + TStringBuf(": ") + Value_; - } - -private: + } + +private: TString Name_; TString Value_; -}; - +}; + /// Контейнер для хранения HTTP-заголовков -class THttpHeaders { +class THttpHeaders { using THeaders = TDeque<THttpInputHeader>; - -public: + +public: using TConstIterator = THeaders::const_iterator; - + THttpHeaders() = default; - /// Добавляет каждую строку из потока в контейнер, считая ее правильным заголовком. + /// Добавляет каждую строку из потока в контейнер, считая ее правильным заголовком. THttpHeaders(IInputStream* stream); - - /// Стандартный итератор. + + /// Стандартный итератор. inline TConstIterator Begin() const noexcept { - return Headers_.begin(); - } + return Headers_.begin(); + } inline TConstIterator begin() const noexcept { return Headers_.begin(); } - - /// Стандартный итератор. + + /// Стандартный итератор. inline TConstIterator End() const noexcept { - return Headers_.end(); - } + return Headers_.end(); + } inline TConstIterator end() const noexcept { return Headers_.end(); } - - /// Возвращает количество заголовков в контейнере. + + /// Возвращает количество заголовков в контейнере. inline size_t Count() const noexcept { - return Headers_.size(); - } - - /// Проверяет, содержит ли контейнер хотя бы один заголовок. + return Headers_.size(); + } + + /// Проверяет, содержит ли контейнер хотя бы один заголовок. inline bool Empty() const noexcept { - return Headers_.empty(); - } + return Headers_.empty(); + } - /// Добавляет заголовок в контейнер. + /// Добавляет заголовок в контейнер. void AddHeader(THttpInputHeader header); - + template <typename ValueType> void AddHeader(TString name, const ValueType& value) { AddHeader(THttpInputHeader(std::move(name), ToString(value))); } - /// Добавляет заголовок в контейнер, если тот не содержит заголовка - /// c таким же параметром. В противном случае, заменяет существующий - /// заголовок на новый. - void AddOrReplaceHeader(const THttpInputHeader& header); - + /// Добавляет заголовок в контейнер, если тот не содержит заголовка + /// c таким же параметром. В противном случае, заменяет существующий + /// заголовок на новый. + void AddOrReplaceHeader(const THttpInputHeader& header); + template <typename ValueType> void AddOrReplaceHeader(TString name, const ValueType& value) { AddOrReplaceHeader(THttpInputHeader(std::move(name), ToString(value))); @@ -109,17 +109,17 @@ public: /// Возвращает nullptr, если не нашел const THttpInputHeader* FindHeader(TStringBuf header) const; - /// Записывает все заголовки контейнера в поток. - /// @details Каждый заголовк записывается в виде "имя параметра: значение\r\n". + /// Записывает все заголовки контейнера в поток. + /// @details Каждый заголовк записывается в виде "имя параметра: значение\r\n". void OutTo(IOutputStream* stream) const; - /// Обменивает наборы заголовков двух контейнеров. + /// Обменивает наборы заголовков двух контейнеров. void Swap(THttpHeaders& headers) noexcept { - Headers_.swap(headers.Headers_); - } - -private: - THeaders Headers_; -}; - + Headers_.swap(headers.Headers_); + } + +private: + THeaders Headers_; +}; + /// @} diff --git a/library/cpp/http/io/headers_ut.cpp b/library/cpp/http/io/headers_ut.cpp index 1d23ef8fdc..b68467f786 100644 --- a/library/cpp/http/io/headers_ut.cpp +++ b/library/cpp/http/io/headers_ut.cpp @@ -13,8 +13,8 @@ namespace { THeadersExistence(const THttpHeaders& headers) { for (THttpHeaders::TConstIterator it = headers.Begin(); - it != headers.End(); - ++it) { + it != headers.End(); + ++it) { Add(it->Name(), it->Value()); } } @@ -24,7 +24,7 @@ namespace { Impl.emplace(TString(name), TString(value)); } - bool operator==(const THeadersExistence& rhs) const { + bool operator==(const THeadersExistence& rhs) const { return Impl == rhs.Impl; } @@ -34,11 +34,11 @@ namespace { }; } -bool operator==(const THeadersExistence& lhs, const THttpHeaders& rhs) { +bool operator==(const THeadersExistence& lhs, const THttpHeaders& rhs) { return lhs == THeadersExistence(rhs); } -bool operator==(const THttpHeaders& lhs, const THeadersExistence& rhs) { +bool operator==(const THttpHeaders& lhs, const THeadersExistence& rhs) { return THeadersExistence(lhs) == rhs; } diff --git a/library/cpp/http/io/list_codings/main.cpp b/library/cpp/http/io/list_codings/main.cpp index 9818d02bdf..f15b24f234 100644 --- a/library/cpp/http/io/list_codings/main.cpp +++ b/library/cpp/http/io/list_codings/main.cpp @@ -1,8 +1,8 @@ #include <library/cpp/http/io/stream.h> -#include <util/stream/output.h> - -int main() { - for (auto codec : SupportedCodings()) { - Cout << codec << Endl; - } -} +#include <util/stream/output.h> + +int main() { + for (auto codec : SupportedCodings()) { + Cout << codec << Endl; + } +} diff --git a/library/cpp/http/io/list_codings/ya.make b/library/cpp/http/io/list_codings/ya.make index e5c5fed6dc..7559ba195c 100644 --- a/library/cpp/http/io/list_codings/ya.make +++ b/library/cpp/http/io/list_codings/ya.make @@ -1,13 +1,13 @@ -PROGRAM() - -OWNER(pg) - -PEERDIR( +PROGRAM() + +OWNER(pg) + +PEERDIR( library/cpp/http/io -) - -SRCS( - main.cpp -) - -END() +) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/http/io/stream.cpp b/library/cpp/http/io/stream.cpp index 6689be684f..c38faffe0b 100644 --- a/library/cpp/http/io/stream.cpp +++ b/library/cpp/http/io/stream.cpp @@ -1,156 +1,156 @@ -#include "stream.h" +#include "stream.h" #include "compression.h" -#include "chunk.h" - +#include "chunk.h" + #include <util/stream/buffered.h> -#include <util/stream/length.h> +#include <util/stream/length.h> #include <util/stream/multi.h> #include <util/stream/null.h> #include <util/stream/tee.h> - + #include <util/system/compat.h> -#include <util/system/yassert.h> - -#include <util/network/socket.h> - -#include <util/string/cast.h> +#include <util/system/yassert.h> + +#include <util/network/socket.h> + +#include <util/string/cast.h> #include <util/string/strip.h> - + #include <util/generic/string.h> -#include <util/generic/utility.h> -#include <util/generic/hash_set.h> -#include <util/generic/yexception.h> - -#define HEADERCMP(header, str) \ - case sizeof(str) - 1: \ +#include <util/generic/utility.h> +#include <util/generic/hash_set.h> +#include <util/generic/yexception.h> + +#define HEADERCMP(header, str) \ + case sizeof(str) - 1: \ if (!stricmp((header).Name().data(), str)) -namespace { +namespace { inline size_t SuggestBufferSize() { - return 8192; - } - + return 8192; + } + inline TStringBuf Trim(const char* b, const char* e) noexcept { - return StripString(TStringBuf(b, e)); - } - + return StripString(TStringBuf(b, e)); + } + inline TStringBuf RmSemiColon(const TStringBuf& s) { - return s.Before(';'); - } - - template <class T, size_t N> - class TStreams: private TNonCopyable { - struct TDelete { - inline void operator()(T* t) noexcept { - delete t; - } - }; - - typedef T* TPtr; - - public: - inline TStreams() noexcept - : Beg_(T_ + N) - { - } - - inline ~TStreams() { - TDelete f; - - ForEach(f); - } - - template <class S> - inline S* Add(S* t) noexcept { - return (S*)AddImpl((T*)t); - } - - template <class Functor> - inline void ForEach(Functor& f) { - const TPtr* end = T_ + N; - - for (TPtr* cur = Beg_; cur != end; ++cur) { - f(*cur); - } - } - - TPtr Top() { - const TPtr* end = T_ + N; - return end == Beg_ ? nullptr : *Beg_; - } - - private: - inline T* AddImpl(T* t) noexcept { - Y_ASSERT(Beg_ > T_); - - return (*--Beg_ = t); - } - - private: - TPtr T_[N]; - TPtr* Beg_; - }; - - template <class TStream> - class TLazy: public IOutputStream { - public: - TLazy(IOutputStream* out, ui16 bs) - : Output_(out) - , BlockSize_(bs) - { - } - - void DoWrite(const void* buf, size_t len) override { - ConstructSlave(); - Slave_->Write(buf, len); - } - - void DoFlush() override { - ConstructSlave(); - Slave_->Flush(); - } - - void DoFinish() override { - ConstructSlave(); - Slave_->Finish(); - } - - private: - inline void ConstructSlave() { - if (!Slave_) { - Slave_.Reset(new TStream(Output_, BlockSize_)); - } - } - - private: - IOutputStream* Output_; - ui16 BlockSize_; - THolder<IOutputStream> Slave_; - }; -} - -class THttpInput::TImpl { + return s.Before(';'); + } + + template <class T, size_t N> + class TStreams: private TNonCopyable { + struct TDelete { + inline void operator()(T* t) noexcept { + delete t; + } + }; + + typedef T* TPtr; + + public: + inline TStreams() noexcept + : Beg_(T_ + N) + { + } + + inline ~TStreams() { + TDelete f; + + ForEach(f); + } + + template <class S> + inline S* Add(S* t) noexcept { + return (S*)AddImpl((T*)t); + } + + template <class Functor> + inline void ForEach(Functor& f) { + const TPtr* end = T_ + N; + + for (TPtr* cur = Beg_; cur != end; ++cur) { + f(*cur); + } + } + + TPtr Top() { + const TPtr* end = T_ + N; + return end == Beg_ ? nullptr : *Beg_; + } + + private: + inline T* AddImpl(T* t) noexcept { + Y_ASSERT(Beg_ > T_); + + return (*--Beg_ = t); + } + + private: + TPtr T_[N]; + TPtr* Beg_; + }; + + template <class TStream> + class TLazy: public IOutputStream { + public: + TLazy(IOutputStream* out, ui16 bs) + : Output_(out) + , BlockSize_(bs) + { + } + + void DoWrite(const void* buf, size_t len) override { + ConstructSlave(); + Slave_->Write(buf, len); + } + + void DoFlush() override { + ConstructSlave(); + Slave_->Flush(); + } + + void DoFinish() override { + ConstructSlave(); + Slave_->Finish(); + } + + private: + inline void ConstructSlave() { + if (!Slave_) { + Slave_.Reset(new TStream(Output_, BlockSize_)); + } + } + + private: + IOutputStream* Output_; + ui16 BlockSize_; + THolder<IOutputStream> Slave_; + }; +} + +class THttpInput::TImpl { typedef THashSet<TString> TAcceptCodings; - -public: + +public: inline TImpl(IInputStream* slave) - : Slave_(slave) - , Buffered_(Slave_, SuggestBufferSize()) + : Slave_(slave) + , Buffered_(Slave_, SuggestBufferSize()) , ChunkedInput_(nullptr) , Input_(nullptr) , FirstLine_(ReadFirstLine(Buffered_)) - , Headers_(&Buffered_) - , KeepAlive_(false) - , HasContentLength_(false) - , ContentLength_(0) - , ContentEncoded_(false) + , Headers_(&Buffered_) + , KeepAlive_(false) + , HasContentLength_(false) + , ContentLength_(0) + , ContentEncoded_(false) , Expect100Continue_(false) - { - BuildInputChain(); + { + BuildInputChain(); Y_ASSERT(Input_); - } - + } + static TString ReadFirstLine(TBufferedInput& in) { TString s; Y_ENSURE_EX(in.ReadLine(s), THttpReadException() << "Failed to get first line"); @@ -158,48 +158,48 @@ public: } inline ~TImpl() { - } - - inline size_t Read(void* buf, size_t len) { + } + + inline size_t Read(void* buf, size_t len) { return Perform(len, [this, buf](size_t toRead) { return Input_->Read(buf, toRead); }); - } + } inline size_t Skip(size_t len) { return Perform(len, [this](size_t toSkip) { return Input_->Skip(toSkip); }); - } - + } + inline const TString& FirstLine() const noexcept { - return FirstLine_; - } - + return FirstLine_; + } + inline const THttpHeaders& Headers() const noexcept { - return Headers_; - } - + return Headers_; + } + inline const TMaybe<THttpHeaders>& Trailers() const noexcept { return Trailers_; } inline bool IsKeepAlive() const noexcept { - return KeepAlive_; - } - + return KeepAlive_; + } + inline bool AcceptEncoding(const TString& s) const { - return Codings_.find(to_lower(s)) != Codings_.end(); - } - + return Codings_.find(to_lower(s)) != Codings_.end(); + } + inline bool GetContentLength(ui64& value) const noexcept { - if (HasContentLength_) { - value = ContentLength_; - return true; + if (HasContentLength_) { + value = ContentLength_; + return true; } - return false; - } + return false; + } inline bool ContentEncoded() const noexcept { - return ContentEncoded_; - } - + return ContentEncoded_; + } + inline bool HasContent() const noexcept { return HasContentLength_ || ChunkedInput_; } @@ -208,8 +208,8 @@ public: return Expect100Continue_; } -private: - template <class Operation> +private: + template <class Operation> inline size_t Perform(size_t len, const Operation& operation) { size_t processed = operation(len); if (processed == 0 && len > 0) { @@ -227,52 +227,52 @@ private: return processed; } - struct TParsedHeaders { + struct TParsedHeaders { bool Chunked = false; bool KeepAlive = false; - TStringBuf LZipped; - }; - - struct TTrEnc { - inline void operator()(const TStringBuf& s) { + TStringBuf LZipped; + }; + + struct TTrEnc { + inline void operator()(const TStringBuf& s) { if (s == TStringBuf("chunked")) { p->Chunked = true; - } - } - - TParsedHeaders* p; - }; - - struct TAccCoding { - inline void operator()(const TStringBuf& s) { - c->insert(ToString(s)); - } - - TAcceptCodings* c; - }; - - template <class Functor> + } + } + + TParsedHeaders* p; + }; + + struct TAccCoding { + inline void operator()(const TStringBuf& s) { + c->insert(ToString(s)); + } + + TAcceptCodings* c; + }; + + template <class Functor> inline void ForEach(TString in, Functor& f) { - in.to_lower(); - - const char* b = in.begin(); - const char* c = b; - const char* e = in.end(); - - while (c != e) { - if (*c == ',') { - f(RmSemiColon(Trim(b, c))); - b = c + 1; - } - - ++c; - } - - if (b != c) { - f(RmSemiColon(Trim(b, c))); - } - } - + in.to_lower(); + + const char* b = in.begin(); + const char* c = b; + const char* e = in.end(); + + while (c != e) { + if (*c == ',') { + f(RmSemiColon(Trim(b, c))); + b = c + 1; + } + + ++c; + } + + if (b != c) { + f(RmSemiColon(Trim(b, c))); + } + } + inline bool IsRequest() const { return strnicmp(FirstLine().data(), "get", 3) == 0 || strnicmp(FirstLine().data(), "post", 4) == 0 || @@ -282,47 +282,47 @@ private: strnicmp(FirstLine().data(), "delete", 6) == 0; } - inline void BuildInputChain() { - TParsedHeaders p; + inline void BuildInputChain() { + TParsedHeaders p; - size_t pos = FirstLine_.rfind(' '); + size_t pos = FirstLine_.rfind(' '); // In HTTP/1.1 Keep-Alive is turned on by default if (pos != TString::npos && strcmp(FirstLine_.c_str() + pos + 1, "HTTP/1.1") == 0) { - p.KeepAlive = true; //request + p.KeepAlive = true; //request } else if (strnicmp(FirstLine_.data(), "HTTP/1.1", 8) == 0) { - p.KeepAlive = true; //reply - } - - for (THttpHeaders::TConstIterator h = Headers_.Begin(); h != Headers_.End(); ++h) { - const THttpInputHeader& header = *h; + p.KeepAlive = true; //reply + } + + for (THttpHeaders::TConstIterator h = Headers_.Begin(); h != Headers_.End(); ++h) { + const THttpInputHeader& header = *h; switch (header.Name().size()) { - HEADERCMP(header, "transfer-encoding") { - TTrEnc f = {&p}; - ForEach(header.Value(), f); - } - break; - HEADERCMP(header, "content-encoding") { - p.LZipped = header.Value(); - } - break; - HEADERCMP(header, "accept-encoding") { - TAccCoding f = {&Codings_}; - ForEach(header.Value(), f); - } - break; - HEADERCMP(header, "content-length") { - HasContentLength_ = true; - ContentLength_ = FromString(header.Value()); - } - break; - HEADERCMP(header, "connection") { - // accept header "Connection: Keep-Alive, TE" + HEADERCMP(header, "transfer-encoding") { + TTrEnc f = {&p}; + ForEach(header.Value(), f); + } + break; + HEADERCMP(header, "content-encoding") { + p.LZipped = header.Value(); + } + break; + HEADERCMP(header, "accept-encoding") { + TAccCoding f = {&Codings_}; + ForEach(header.Value(), f); + } + break; + HEADERCMP(header, "content-length") { + HasContentLength_ = true; + ContentLength_ = FromString(header.Value()); + } + break; + HEADERCMP(header, "connection") { + // accept header "Connection: Keep-Alive, TE" if (strnicmp(header.Value().data(), "keep-alive", 10) == 0) { p.KeepAlive = true; } else if (stricmp(header.Value().data(), "close") == 0) { p.KeepAlive = false; } - } + } [[fallthrough]]; HEADERCMP(header, "expect") { auto findContinue = [&](const TStringBuf& s) { @@ -332,101 +332,101 @@ private: }; ForEach(header.Value(), findContinue); } - break; - } - } - + break; + } + } + if (p.Chunked) { ChunkedInput_ = Streams_.Add(new TChunkedInput(&Buffered_, &Trailers_)); Input_ = ChunkedInput_; - } else { + } else { // disable buffering - Buffered_.Reset(&Cnull); - Input_ = Streams_.Add(new TMultiInput(&Buffered_, Slave_)); - + Buffered_.Reset(&Cnull); + Input_ = Streams_.Add(new TMultiInput(&Buffered_, Slave_)); + if (IsRequest() || HasContentLength_) { - /* - * TODO - we have other cases - */ - Input_ = Streams_.Add(new TLengthLimitedInput(Input_, ContentLength_)); - } - } - + /* + * TODO - we have other cases + */ + Input_ = Streams_.Add(new TLengthLimitedInput(Input_, ContentLength_)); + } + } + if (auto decoder = TCompressionCodecFactory::Instance().FindDecoder(p.LZipped)) { - ContentEncoded_ = true; + ContentEncoded_ = true; Input_ = Streams_.Add((*decoder)(Input_).Release()); - } - + } + KeepAlive_ = p.KeepAlive; - } - -private: + } + +private: IInputStream* Slave_; - - /* - * input helpers - */ - TBufferedInput Buffered_; + + /* + * input helpers + */ + TBufferedInput Buffered_; TStreams<IInputStream, 8> Streams_; IInputStream* ChunkedInput_; - - /* - * final input stream - */ + + /* + * final input stream + */ IInputStream* Input_; - + TString FirstLine_; - THttpHeaders Headers_; + THttpHeaders Headers_; TMaybe<THttpHeaders> Trailers_; - bool KeepAlive_; + bool KeepAlive_; + + TAcceptCodings Codings_; - TAcceptCodings Codings_; + bool HasContentLength_; + ui64 ContentLength_; - bool HasContentLength_; - ui64 ContentLength_; - - bool ContentEncoded_; + bool ContentEncoded_; bool Expect100Continue_; -}; - +}; + THttpInput::THttpInput(IInputStream* slave) - : Impl_(new TImpl(slave)) -{ -} - + : Impl_(new TImpl(slave)) +{ +} + THttpInput::THttpInput(THttpInput&& httpInput) = default; THttpInput::~THttpInput() { -} - -size_t THttpInput::DoRead(void* buf, size_t len) { - return Impl_->Read(buf, len); -} - +} + +size_t THttpInput::DoRead(void* buf, size_t len) { + return Impl_->Read(buf, len); +} + size_t THttpInput::DoSkip(size_t len) { return Impl_->Skip(len); } const THttpHeaders& THttpInput::Headers() const noexcept { - return Impl_->Headers(); -} - + return Impl_->Headers(); +} + const TMaybe<THttpHeaders>& THttpInput::Trailers() const noexcept { return Impl_->Trailers(); } const TString& THttpInput::FirstLine() const noexcept { - return Impl_->FirstLine(); -} - + return Impl_->FirstLine(); +} + bool THttpInput::IsKeepAlive() const noexcept { - return Impl_->IsKeepAlive(); -} - + return Impl_->IsKeepAlive(); +} + bool THttpInput::AcceptEncoding(const TString& coding) const { - return Impl_->AcceptEncoding(coding); -} - + return Impl_->AcceptEncoding(coding); +} + TString THttpInput::BestCompressionScheme(TArrayRef<const TStringBuf> codings) const { return NHttp::ChooseBestCompressionScheme( [this](const TString& coding) { @@ -434,8 +434,8 @@ TString THttpInput::BestCompressionScheme(TArrayRef<const TStringBuf> codings) c }, codings ); -} - +} + TString THttpInput::BestCompressionScheme() const { return BestCompressionScheme(TCompressionCodecFactory::Instance().GetBestCodecs()); } @@ -456,141 +456,141 @@ bool THttpInput::HasExpect100Continue() const noexcept { return Impl_->HasExpect100Continue(); } -class THttpOutput::TImpl { +class THttpOutput::TImpl { class TSizeCalculator: public IOutputStream { - public: + public: inline TSizeCalculator() noexcept { - } - + } + ~TSizeCalculator() override { - } - + } + void DoWrite(const void* /*buf*/, size_t len) override { - Length_ += len; - } - + Length_ += len; + } + inline size_t Length() const noexcept { - return Length_; - } - - private: + return Length_; + } + + private: size_t Length_ = 0; - }; - - enum TState { - Begin = 0, - FirstLineSent = 1, - HeadersSent = 2 - }; - - struct TFlush { + }; + + enum TState { + Begin = 0, + FirstLineSent = 1, + HeadersSent = 2 + }; + + struct TFlush { inline void operator()(IOutputStream* s) { - s->Flush(); - } - }; - - struct TFinish { + s->Flush(); + } + }; + + struct TFinish { inline void operator()(IOutputStream* s) { - s->Finish(); - } - }; - -public: + s->Finish(); + } + }; + +public: inline TImpl(IOutputStream* slave, THttpInput* request) - : Slave_(slave) - , State_(Begin) - , Output_(Slave_) - , Request_(request) - , Version_(1100) - , KeepAliveEnabled_(false) + : Slave_(slave) + , State_(Begin) + , Output_(Slave_) + , Request_(request) + , Version_(1100) + , KeepAliveEnabled_(false) , BodyEncodingEnabled_(true) , CompressionHeaderEnabled_(true) - , Finished_(false) - { - } - + , Finished_(false) + { + } + inline ~TImpl() { - } - + } + inline void SendContinue() { Output_->Write("HTTP/1.1 100 Continue\r\n\r\n"); Output_->Flush(); } - inline void Write(const void* buf, size_t len) { - if (Finished_) { - ythrow THttpException() << "can not write to finished stream"; - } - - if (State_ == HeadersSent) { - Output_->Write(buf, len); - - return; - } - - const char* b = (const char*)buf; - const char* e = b + len; - const char* c = b; - - while (c != e) { - if (*c == '\n') { - Line_.append(b, c); - + inline void Write(const void* buf, size_t len) { + if (Finished_) { + ythrow THttpException() << "can not write to finished stream"; + } + + if (State_ == HeadersSent) { + Output_->Write(buf, len); + + return; + } + + const char* b = (const char*)buf; + const char* e = b + len; + const char* c = b; + + while (c != e) { + if (*c == '\n') { + Line_.append(b, c); + if (!Line_.empty() && Line_.back() == '\r') { - Line_.pop_back(); - } - - b = c + 1; - - Process(Line_); - - if (State_ == HeadersSent) { - Output_->Write(b, e - b); - - return; - } - - Line_.clear(); - } - - ++c; - } - - if (b != c) { - Line_.append(b, c); - } - } - - inline void Flush() { - TFlush f; - Streams_.ForEach(f); + Line_.pop_back(); + } + + b = c + 1; + + Process(Line_); + + if (State_ == HeadersSent) { + Output_->Write(b, e - b); + + return; + } + + Line_.clear(); + } + + ++c; + } + + if (b != c) { + Line_.append(b, c); + } + } + + inline void Flush() { + TFlush f; + Streams_.ForEach(f); Slave_->Flush(); // see SEARCH-1030 - } - - inline void Finish() { - if (Finished_) { - return; - } - - TFinish f; - Streams_.ForEach(f); + } + + inline void Finish() { + if (Finished_) { + return; + } + + TFinish f; + Streams_.ForEach(f); Slave_->Finish(); // see SEARCH-1030 - - Finished_ = true; - } - + + Finished_ = true; + } + inline const THttpHeaders& SentHeaders() const noexcept { - return Headers_; - } - + return Headers_; + } + inline void EnableCompression(TArrayRef<const TStringBuf> schemas) { - ComprSchemas_ = schemas; - } - - inline void EnableKeepAlive(bool enable) { - KeepAliveEnabled_ = enable; - } - + ComprSchemas_ = schemas; + } + + inline void EnableKeepAlive(bool enable) { + KeepAliveEnabled_ = enable; + } + inline void EnableBodyEncoding(bool enable) { BodyEncodingEnabled_ = enable; } @@ -601,12 +601,12 @@ public: inline bool IsCompressionEnabled() const noexcept { return !ComprSchemas_.empty(); - } - + } + inline bool IsKeepAliveEnabled() const noexcept { - return KeepAliveEnabled_; - } - + return KeepAliveEnabled_; + } + inline bool IsBodyEncodingEnabled() const noexcept { return BodyEncodingEnabled_; } @@ -616,9 +616,9 @@ public: } inline bool CanBeKeepAlive() const noexcept { - return SupportChunkedTransfer() && IsKeepAliveEnabled() && (Request_ ? Request_->IsKeepAlive() : true); - } - + return SupportChunkedTransfer() && IsKeepAliveEnabled() && (Request_ ? Request_->IsKeepAlive() : true); + } + inline const TString& FirstLine() const noexcept { return FirstLine_; } @@ -627,18 +627,18 @@ public: return SizeCalculator_.Length(); } -private: +private: static inline bool IsResponse(const TString& s) noexcept { return strnicmp(s.data(), "HTTP/", 5) == 0; - } - + } + static inline bool IsRequest(const TString& s) noexcept { - return !IsResponse(s); - } - + return !IsResponse(s); + } + inline bool IsHttpRequest() const noexcept { - return IsRequest(FirstLine_); - } + return IsRequest(FirstLine_); + } inline bool HasResponseBody() const noexcept { if (IsHttpResponse()) { @@ -652,169 +652,169 @@ private: } inline bool IsHttpResponse() const noexcept { - return IsResponse(FirstLine_); - } - + return IsResponse(FirstLine_); + } + inline bool HasRequestBody() const noexcept { return strnicmp(FirstLine_.data(), "POST", 4) == 0 || strnicmp(FirstLine_.data(), "PATCH", 5) == 0 || strnicmp(FirstLine_.data(), "PUT", 3) == 0; - } + } static inline size_t ParseHttpVersion(const TString& s) { - if (s.empty()) { - ythrow THttpParseException() << "malformed http stream"; + if (s.empty()) { + ythrow THttpParseException() << "malformed http stream"; } - - size_t parsed_version = 0; - - if (IsResponse(s)) { + + size_t parsed_version = 0; + + if (IsResponse(s)) { const char* b = s.data() + 5; - - while (*b && *b != ' ') { - if (*b != '.') { - parsed_version *= 10; - parsed_version += (*b - '0'); - } - - ++b; - } - } else { - /* - * s not empty here - */ - const char* e = s.end() - 1; - const char* b = s.begin(); - size_t mult = 1; - - while (e != b && *e != '/') { - if (*e != '.') { - parsed_version += (*e - '0') * mult; - mult *= 10; - } - - --e; - } - } - - return parsed_version * 100; - } - - inline void ParseHttpVersion() { - size_t parsed_version = ParseHttpVersion(FirstLine_); - - if (Request_) { - parsed_version = Min(parsed_version, ParseHttpVersion(Request_->FirstLine())); - } - - Version_ = parsed_version; - } - + + while (*b && *b != ' ') { + if (*b != '.') { + parsed_version *= 10; + parsed_version += (*b - '0'); + } + + ++b; + } + } else { + /* + * s not empty here + */ + const char* e = s.end() - 1; + const char* b = s.begin(); + size_t mult = 1; + + while (e != b && *e != '/') { + if (*e != '.') { + parsed_version += (*e - '0') * mult; + mult *= 10; + } + + --e; + } + } + + return parsed_version * 100; + } + + inline void ParseHttpVersion() { + size_t parsed_version = ParseHttpVersion(FirstLine_); + + if (Request_) { + parsed_version = Min(parsed_version, ParseHttpVersion(Request_->FirstLine())); + } + + Version_ = parsed_version; + } + inline void Process(const TString& s) { Y_ASSERT(State_ != HeadersSent); - - if (State_ == Begin) { - FirstLine_ = s; - ParseHttpVersion(); - State_ = FirstLineSent; - } else { - if (s.empty()) { - BuildOutputStream(); - WriteCached(); - State_ = HeadersSent; - } else { + + if (State_ == Begin) { + FirstLine_ = s; + ParseHttpVersion(); + State_ = FirstLineSent; + } else { + if (s.empty()) { + BuildOutputStream(); + WriteCached(); + State_ = HeadersSent; + } else { AddHeader(THttpInputHeader(s)); - } - } - } - + } + } + } + inline void WriteCachedImpl(IOutputStream* s) const { s->Write(FirstLine_.data(), FirstLine_.size()); - s->Write("\r\n", 2); - Headers_.OutTo(s); - s->Write("\r\n", 2); - s->Finish(); - } - - inline void WriteCached() { - size_t buflen = 0; - - { - TSizeCalculator out; - - WriteCachedImpl(&out); - buflen = out.Length(); - } - - { - TBufferedOutput out(Slave_, buflen); - - WriteCachedImpl(&out); - } - - if (IsHttpRequest() && !HasRequestBody()) { - /* - * if this is http request, then send it now - */ - - Slave_->Flush(); - } - } - + s->Write("\r\n", 2); + Headers_.OutTo(s); + s->Write("\r\n", 2); + s->Finish(); + } + + inline void WriteCached() { + size_t buflen = 0; + + { + TSizeCalculator out; + + WriteCachedImpl(&out); + buflen = out.Length(); + } + + { + TBufferedOutput out(Slave_, buflen); + + WriteCachedImpl(&out); + } + + if (IsHttpRequest() && !HasRequestBody()) { + /* + * if this is http request, then send it now + */ + + Slave_->Flush(); + } + } + inline bool SupportChunkedTransfer() const noexcept { - return Version_ >= 1100; - } - - inline void BuildOutputStream() { - if (CanBeKeepAlive()) { - AddOrReplaceHeader(THttpInputHeader("Connection", "Keep-Alive")); - } else { - AddOrReplaceHeader(THttpInputHeader("Connection", "Close")); - } - - if (IsHttpResponse()) { + return Version_ >= 1100; + } + + inline void BuildOutputStream() { + if (CanBeKeepAlive()) { + AddOrReplaceHeader(THttpInputHeader("Connection", "Keep-Alive")); + } else { + AddOrReplaceHeader(THttpInputHeader("Connection", "Close")); + } + + if (IsHttpResponse()) { if (Request_ && IsCompressionEnabled() && HasResponseBody()) { TString scheme = Request_->BestCompressionScheme(ComprSchemas_); - if (scheme != "identity") { - AddOrReplaceHeader(THttpInputHeader("Content-Encoding", scheme)); + if (scheme != "identity") { + AddOrReplaceHeader(THttpInputHeader("Content-Encoding", scheme)); RemoveHeader("Content-Length"); - } - } - - RebuildStream(); - } else { - if (IsCompressionEnabled()) { - AddOrReplaceHeader(THttpInputHeader("Accept-Encoding", BuildAcceptEncoding())); - } - if (HasRequestBody()) { - RebuildStream(); - } - } - } - + } + } + + RebuildStream(); + } else { + if (IsCompressionEnabled()) { + AddOrReplaceHeader(THttpInputHeader("Accept-Encoding", BuildAcceptEncoding())); + } + if (HasRequestBody()) { + RebuildStream(); + } + } + } + inline TString BuildAcceptEncoding() const { TString ret; - + for (const auto& coding : ComprSchemas_) { - if (ret) { - ret += ", "; - } - + if (ret) { + ret += ", "; + } + ret += coding; - } - - return ret; - } + } + + return ret; + } - inline void RebuildStream() { + inline void RebuildStream() { bool keepAlive = false; const TCompressionCodecFactory::TEncoderConstructor* encoder = nullptr; - bool chunked = false; + bool chunked = false; bool haveContentLength = false; - - for (THttpHeaders::TConstIterator h = Headers_.Begin(); h != Headers_.End(); ++h) { - const THttpInputHeader& header = *h; + + for (THttpHeaders::TConstIterator h = Headers_.Begin(); h != Headers_.End(); ++h) { + const THttpInputHeader& header = *h; const TString hl = to_lower(header.Name()); - + if (hl == TStringBuf("connection")) { keepAlive = to_lower(header.Value()) == TStringBuf("keep-alive"); } else if (IsCompressionHeaderEnabled() && hl == TStringBuf("content-encoding")) { @@ -823,109 +823,109 @@ private: chunked = to_lower(header.Value()) == TStringBuf("chunked"); } else if (hl == TStringBuf("content-length")) { haveContentLength = true; - } - } - + } + } + if (!haveContentLength && !chunked && (IsHttpRequest() || HasResponseBody()) && SupportChunkedTransfer() && (keepAlive || encoder || IsHttpRequest())) { - AddHeader(THttpInputHeader("Transfer-Encoding", "chunked")); - chunked = true; - } - + AddHeader(THttpInputHeader("Transfer-Encoding", "chunked")); + chunked = true; + } + if (IsBodyEncodingEnabled() && chunked) { - Output_ = Streams_.Add(new TChunkedOutput(Output_)); - } - + Output_ = Streams_.Add(new TChunkedOutput(Output_)); + } + Output_ = Streams_.Add(new TTeeOutput(Output_, &SizeCalculator_)); if (IsBodyEncodingEnabled() && encoder) { Output_ = Streams_.Add((*encoder)(Output_).Release()); - } - } - - inline void AddHeader(const THttpInputHeader& hdr) { - Headers_.AddHeader(hdr); - } - - inline void AddOrReplaceHeader(const THttpInputHeader& hdr) { - Headers_.AddOrReplaceHeader(hdr); - } - + } + } + + inline void AddHeader(const THttpInputHeader& hdr) { + Headers_.AddHeader(hdr); + } + + inline void AddOrReplaceHeader(const THttpInputHeader& hdr) { + Headers_.AddOrReplaceHeader(hdr); + } + inline void RemoveHeader(const TString& hdr) { Headers_.RemoveHeader(hdr); } -private: +private: IOutputStream* Slave_; - TState State_; + TState State_; IOutputStream* Output_; TStreams<IOutputStream, 8> Streams_; TString Line_; TString FirstLine_; - THttpHeaders Headers_; - THttpInput* Request_; - size_t Version_; - + THttpHeaders Headers_; + THttpInput* Request_; + size_t Version_; + TArrayRef<const TStringBuf> ComprSchemas_; - - bool KeepAliveEnabled_; + + bool KeepAliveEnabled_; bool BodyEncodingEnabled_; bool CompressionHeaderEnabled_; - bool Finished_; + bool Finished_; TSizeCalculator SizeCalculator_; -}; - +}; + THttpOutput::THttpOutput(IOutputStream* slave) : Impl_(new TImpl(slave, nullptr)) -{ -} - +{ +} + THttpOutput::THttpOutput(IOutputStream* slave, THttpInput* request) - : Impl_(new TImpl(slave, request)) -{ -} - + : Impl_(new TImpl(slave, request)) +{ +} + THttpOutput::~THttpOutput() { - try { - Finish(); - } catch (...) { - } -} - -void THttpOutput::DoWrite(const void* buf, size_t len) { - Impl_->Write(buf, len); -} - -void THttpOutput::DoFlush() { - Impl_->Flush(); -} - -void THttpOutput::DoFinish() { - Impl_->Finish(); -} - + try { + Finish(); + } catch (...) { + } +} + +void THttpOutput::DoWrite(const void* buf, size_t len) { + Impl_->Write(buf, len); +} + +void THttpOutput::DoFlush() { + Impl_->Flush(); +} + +void THttpOutput::DoFinish() { + Impl_->Finish(); +} + const THttpHeaders& THttpOutput::SentHeaders() const noexcept { return Impl_->SentHeaders(); -} - -void THttpOutput::EnableCompression(bool enable) { - if (enable) { +} + +void THttpOutput::EnableCompression(bool enable) { + if (enable) { EnableCompression(TCompressionCodecFactory::Instance().GetBestCodecs()); - } else { + } else { TArrayRef<TStringBuf> codings; EnableCompression(codings); - } -} - + } +} + void THttpOutput::EnableCompression(TArrayRef<const TStringBuf> schemas) { Impl_->EnableCompression(schemas); -} - -void THttpOutput::EnableKeepAlive(bool enable) { - Impl_->EnableKeepAlive(enable); -} - +} + +void THttpOutput::EnableKeepAlive(bool enable) { + Impl_->EnableKeepAlive(enable); +} + void THttpOutput::EnableBodyEncoding(bool enable) { Impl_->EnableBodyEncoding(enable); } @@ -935,25 +935,25 @@ void THttpOutput::EnableCompressionHeader(bool enable) { } bool THttpOutput::IsKeepAliveEnabled() const noexcept { - return Impl_->IsKeepAliveEnabled(); -} - + return Impl_->IsKeepAliveEnabled(); +} + bool THttpOutput::IsBodyEncodingEnabled() const noexcept { return Impl_->IsBodyEncodingEnabled(); } bool THttpOutput::IsCompressionEnabled() const noexcept { - return Impl_->IsCompressionEnabled(); -} - + return Impl_->IsCompressionEnabled(); +} + bool THttpOutput::IsCompressionHeaderEnabled() const noexcept { return Impl_->IsCompressionHeaderEnabled(); } bool THttpOutput::CanBeKeepAlive() const noexcept { - return Impl_->CanBeKeepAlive(); -} - + return Impl_->CanBeKeepAlive(); +} + void THttpOutput::SendContinue() { Impl_->SendContinue(); } @@ -966,13 +966,13 @@ size_t THttpOutput::SentSize() const noexcept { return Impl_->SentSize(); } -unsigned ParseHttpRetCode(const TStringBuf& ret) { - const TStringBuf code = StripString(StripString(ret.After(' ')).Before(' ')); - +unsigned ParseHttpRetCode(const TStringBuf& ret) { + const TStringBuf code = StripString(StripString(ret.After(' ')).Before(' ')); + return FromString<unsigned>(code.data(), code.size()); -} +} -void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf& request, const TStringBuf& agent, const TStringBuf& from) { +void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf& request, const TStringBuf& agent, const TStringBuf& from) { TSocketOutput so(s); THttpOutput output(&so); @@ -995,11 +995,11 @@ void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf IOutputStream::TPart::CrLf(), IOutputStream::TPart::CrLf(), }; - + output.Write(parts, sizeof(parts) / sizeof(*parts)); output.Finish(); } - + TArrayRef<const TStringBuf> SupportedCodings() { return TCompressionCodecFactory::Instance().GetBestCodecs(); -} +} diff --git a/library/cpp/http/io/stream.h b/library/cpp/http/io/stream.h index 78ca4fc814..ab69a8abab 100644 --- a/library/cpp/http/io/stream.h +++ b/library/cpp/http/io/stream.h @@ -1,40 +1,40 @@ #pragma once - -#include "headers.h" - + +#include "headers.h" + #include <util/stream/output.h> #include <util/generic/maybe.h> #include <util/generic/ptr.h> #include <util/generic/string.h> -#include <util/generic/strbuf.h> +#include <util/generic/strbuf.h> #include <util/generic/yexception.h> -#include <util/generic/array_ref.h> - -class TSocket; - -struct THttpException: public yexception { -}; - -struct THttpParseException: public THttpException { -}; - +#include <util/generic/array_ref.h> + +class TSocket; + +struct THttpException: public yexception { +}; + +struct THttpParseException: public THttpException { +}; + struct THttpReadException: public THttpException { }; /// Чтение ответа HTTP-сервера. class THttpInput: public IInputStream { -public: +public: THttpInput(IInputStream* slave); THttpInput(THttpInput&& httpInput); ~THttpInput() override; - - /* - * parsed http headers - */ - /// Возвращает контейнер с заголовками ответа HTTP-сервера. + + /* + * parsed http headers + */ + /// Возвращает контейнер с заголовками ответа HTTP-сервера. const THttpHeaders& Headers() const noexcept; - - /* + + /* * parsed http trailers */ /// Возвращает контейнер (возможно пустой) с trailer'ами ответа HTTP-сервера. @@ -44,78 +44,78 @@ public: const TMaybe<THttpHeaders>& Trailers() const noexcept; /* - * first line - response or request - */ - /// Возвращает первую строку ответа HTTP-сервера. - /// @details Первая строка HTTP-сервера - строка состояния, - /// содержащая три поля: версию HTTP, код состояния и описание. + * first line - response or request + */ + /// Возвращает первую строку ответа HTTP-сервера. + /// @details Первая строка HTTP-сервера - строка состояния, + /// содержащая три поля: версию HTTP, код состояния и описание. const TString& FirstLine() const noexcept; - - /* - * connection can be keep-alive - */ - /// Проверяет, не завершено ли соединение с сервером. - /// @details Транзакция считается завершенной, если не передан заголовок - /// "Connection: Keep Alive". + + /* + * connection can be keep-alive + */ + /// Проверяет, не завершено ли соединение с сервером. + /// @details Транзакция считается завершенной, если не передан заголовок + /// "Connection: Keep Alive". bool IsKeepAlive() const noexcept; - - /* - * output data can be encoded - */ - /// Проверяет, поддерживается ли данный тип кодирования содержимого - /// ответа HTTP-сервера. + + /* + * output data can be encoded + */ + /// Проверяет, поддерживается ли данный тип кодирования содержимого + /// ответа HTTP-сервера. bool AcceptEncoding(const TString& coding) const; - /// Пытается определить наилучший тип кодирования ответа HTTP-сервера. - /// @details Если ответ сервера говорит о том, что поддерживаются - /// любые типы кодирования, выбирается gzip. В противном случае - /// из списка типов кодирования выбирается лучший из поддерживаемых сервером. + /// Пытается определить наилучший тип кодирования ответа HTTP-сервера. + /// @details Если ответ сервера говорит о том, что поддерживаются + /// любые типы кодирования, выбирается gzip. В противном случае + /// из списка типов кодирования выбирается лучший из поддерживаемых сервером. TString BestCompressionScheme() const; TString BestCompressionScheme(TArrayRef<const TStringBuf> codings) const; - - /// Если заголовки содержат Content-Length, возвращает true и - /// записывает значение из заголовка в value + + /// Если заголовки содержат Content-Length, возвращает true и + /// записывает значение из заголовка в value bool GetContentLength(ui64& value) const noexcept; - /// Признак запакованности данных, - если выставлен, то Content-Length, при наличии в заголовках, - /// показывает объём запакованных данных, а из THttpInput мы будем вычитывать уже распакованные. + /// Признак запакованности данных, - если выставлен, то Content-Length, при наличии в заголовках, + /// показывает объём запакованных данных, а из THttpInput мы будем вычитывать уже распакованные. bool ContentEncoded() const noexcept; - + /// Returns true if Content-Length or Transfer-Encoding header received bool HasContent() const noexcept; bool HasExpect100Continue() const noexcept; -private: +private: size_t DoRead(void* buf, size_t len) override; size_t DoSkip(size_t len) override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /// Передача запроса HTTP-серверу. class THttpOutput: public IOutputStream { -public: +public: THttpOutput(IOutputStream* slave); THttpOutput(IOutputStream* slave, THttpInput* request); ~THttpOutput() override; - - /* - * sent http headers - */ - /// Возвращает контейнер с заголовками запроса к HTTP-серверу. + + /* + * sent http headers + */ + /// Возвращает контейнер с заголовками запроса к HTTP-серверу. const THttpHeaders& SentHeaders() const noexcept; - - /// Устанавливает режим, при котором сервер выдает ответ в упакованном виде. - void EnableCompression(bool enable); + + /// Устанавливает режим, при котором сервер выдает ответ в упакованном виде. + void EnableCompression(bool enable); void EnableCompression(TArrayRef<const TStringBuf> schemas); - /// Устанавливает режим, при котором соединение с сервером не завершается - /// после окончания транзакции. - void EnableKeepAlive(bool enable); - + /// Устанавливает режим, при котором соединение с сервером не завершается + /// после окончания транзакции. + void EnableKeepAlive(bool enable); + /// Устанавливает режим, при котором тело HTTP-запроса/ответа преобразуется в соответствии /// с заголовками Content-Encoding и Transfer-Encoding (включен по умолчанию) void EnableBodyEncoding(bool enable); @@ -124,12 +124,12 @@ public: /// указанным в Content-Encoding (включен по умолчанию) void EnableCompressionHeader(bool enable); - /// Проверяет, производится ли выдача ответов в упакованном виде. + /// Проверяет, производится ли выдача ответов в упакованном виде. bool IsCompressionEnabled() const noexcept; - /// Проверяет, не завершается ли соединение с сервером после окончания транзакции. + /// Проверяет, не завершается ли соединение с сервером после окончания транзакции. bool IsKeepAliveEnabled() const noexcept; - + /// Проверяет, преобразуется ли тело HTTP-запроса/ответа в соответствии /// с заголовками Content-Encoding и Transfer-Encoding bool IsBodyEncodingEnabled() const noexcept; @@ -138,13 +138,13 @@ public: /// указанным в Content-Encoding bool IsCompressionHeaderEnabled() const noexcept; - /* - * is this connection can be really keep-alive - */ - /// Проверяет, можно ли установить режим, при котором соединение с сервером - /// не завершается после окончания транзакции. + /* + * is this connection can be really keep-alive + */ + /// Проверяет, можно ли установить режим, при котором соединение с сервером + /// не завершается после окончания транзакции. bool CanBeKeepAlive() const noexcept; - + void SendContinue(); /* @@ -157,22 +157,22 @@ public: /// учёта chunked transfer encoding) size_t SentSize() const noexcept; -private: +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /// Возвращает код состояния из ответа сервера. -unsigned ParseHttpRetCode(const TStringBuf& ret); - +unsigned ParseHttpRetCode(const TStringBuf& ret); + /// Отправляет HTTP-серверу запрос с минимумом необходимых заголовков. -void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf& request, const TStringBuf& agent = "YandexSomething/1.0", const TStringBuf& from = "webadmin@yandex.ru"); +void SendMinimalHttpRequest(TSocket& s, const TStringBuf& host, const TStringBuf& request, const TStringBuf& agent = "YandexSomething/1.0", const TStringBuf& from = "webadmin@yandex.ru"); TArrayRef<const TStringBuf> SupportedCodings(); - + /// @} diff --git a/library/cpp/http/io/stream_ut.cpp b/library/cpp/http/io/stream_ut.cpp index 1ea35df675..b597320405 100644 --- a/library/cpp/http/io/stream_ut.cpp +++ b/library/cpp/http/io/stream_ut.cpp @@ -1,19 +1,19 @@ -#include "stream.h" +#include "stream.h" #include "chunk.h" - + #include <library/cpp/http/server/http_ex.h> - + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - -#include <util/string/printf.h> -#include <util/network/socket.h> + +#include <util/string/printf.h> +#include <util/network/socket.h> #include <util/stream/file.h> #include <util/stream/output.h> #include <util/stream/tee.h> -#include <util/stream/zlib.h> +#include <util/stream/zlib.h> #include <util/stream/null.h> - + Y_UNIT_TEST_SUITE(THttpStreamTest) { class TTestHttpServer: public THttpServer::ICallBack { class TRequest: public THttpClientRequestEx { @@ -34,12 +34,12 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { // "lo" is for "local" if (RD.ServerName() == "yandex.lo") { // do redirect - Output() << "HTTP/1.1 301 Moved permanently\r\n" - "Location: http://www.yandex.lo\r\n" - "\r\n"; + Output() << "HTTP/1.1 301 Moved permanently\r\n" + "Location: http://www.yandex.lo\r\n" + "\r\n"; } else if (RD.ServerName() == "www.yandex.lo") { - Output() << "HTTP/1.1 200 Ok\r\n" - "\r\n"; + Output() << "HTTP/1.1 200 Ok\r\n" + "\r\n"; } else { Output() << "HTTP/1.1 200 Ok\r\n\r\n"; if (Buf.Size()) { @@ -80,7 +80,7 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { Y_UNIT_TEST(TestCodings1) { UNIT_ASSERT(SupportedCodings().size() > 0); - } + } Y_UNIT_TEST(TestHttpInput) { TString res = "I'm a teapot"; @@ -93,41 +93,41 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { UNIT_ASSERT(server.Start()); TNetworkAddress addr("localhost", port); - TSocket s(addr); - - //TDebugOutput dbg; - TNullOutput dbg; - - { - TSocketOutput so(s); - TTeeOutput out(&so, &dbg); - THttpOutput output(&out); - - output.EnableKeepAlive(true); - output.EnableCompression(true); - + TSocket s(addr); + + //TDebugOutput dbg; + TNullOutput dbg; + + { + TSocketOutput so(s); + TTeeOutput out(&so, &dbg); + THttpOutput output(&out); + + output.EnableKeepAlive(true); + output.EnableCompression(true); + TString r; - r += "GET / HTTP/1.1"; - r += "\r\n"; + r += "GET / HTTP/1.1"; + r += "\r\n"; r += "Host: yandex.lo"; - r += "\r\n"; - r += "\r\n"; - + r += "\r\n"; + r += "\r\n"; + output.Write(r.data(), r.size()); - output.Finish(); - } - - { - TSocketInput si(s); - THttpInput input(&si); - unsigned httpCode = ParseHttpRetCode(input.FirstLine()); - UNIT_ASSERT_VALUES_EQUAL(httpCode / 10, 30u); - - TransferData(&input, &dbg); - } + output.Finish(); + } + + { + TSocketInput si(s); + THttpInput input(&si); + unsigned httpCode = ParseHttpRetCode(input.FirstLine()); + UNIT_ASSERT_VALUES_EQUAL(httpCode / 10, 30u); + + TransferData(&input, &dbg); + } server.Stop(); } - + Y_UNIT_TEST(TestHttpInputDelete) { TString res = "I'm a teapot"; TPortManager pm; @@ -175,9 +175,9 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { } Y_UNIT_TEST(TestParseHttpRetCode) { - UNIT_ASSERT_VALUES_EQUAL(ParseHttpRetCode("HTTP/1.1 301"), 301u); - } - + UNIT_ASSERT_VALUES_EQUAL(ParseHttpRetCode("HTTP/1.1 301"), 301u); + } + Y_UNIT_TEST(TestKeepAlive) { { TString s = "GET / HTTP/1.0\r\n\r\n"; @@ -248,19 +248,19 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { TNetworkAddress addr("localhost", port); - TSocket s(addr); - TNullOutput dbg; - + TSocket s(addr); + TNullOutput dbg; + SendMinimalHttpRequest(s, "www.yandex.lo", "/"); - + TSocketInput si(s); - THttpInput input(&si); + THttpInput input(&si); unsigned httpCode = ParseHttpRetCode(input.FirstLine()); - UNIT_ASSERT_VALUES_EQUAL(httpCode, 200u); - - TransferData(&input, &dbg); + UNIT_ASSERT_VALUES_EQUAL(httpCode, 200u); + + TransferData(&input, &dbg); server.Stop(); - } + } Y_UNIT_TEST(TestResponseWithBlanks) { TString res = "qqqqqq\r\n\r\nsdasdsad\r\n"; @@ -276,7 +276,7 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { TSocket s(addr); - SendMinimalHttpRequest(s, "www.yandex.ru", "/"); + SendMinimalHttpRequest(s, "www.yandex.ru", "/"); TSocketInput si(s); THttpInput input(&si); @@ -289,70 +289,70 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { Y_UNIT_TEST(TestOutputFlush) { TString str; - TStringOutput strOut(str); - TBufferedOutput bufOut(&strOut, 8192); - THttpOutput httpOut(&bufOut); - - httpOut.EnableKeepAlive(true); - httpOut.EnableCompression(true); + TStringOutput strOut(str); + TBufferedOutput bufOut(&strOut, 8192); + THttpOutput httpOut(&bufOut); - const char* header = "GET / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; - httpOut << header; + httpOut.EnableKeepAlive(true); + httpOut.EnableCompression(true); + + const char* header = "GET / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; + httpOut << header; unsigned curLen = str.size(); - const char* body = "<html>Hello</html>"; - httpOut << body; + const char* body = "<html>Hello</html>"; + httpOut << body; UNIT_ASSERT_VALUES_EQUAL(curLen, str.size()); - httpOut.Flush(); + httpOut.Flush(); UNIT_ASSERT_VALUES_EQUAL(curLen + strlen(body), str.size()); - } + } Y_UNIT_TEST(TestOutputPostFlush) { TString str; TString checkStr; - TStringOutput strOut(str); - TStringOutput checkOut(checkStr); - TBufferedOutput bufOut(&strOut, 8192); - TTeeOutput teeOut(&bufOut, &checkOut); - THttpOutput httpOut(&teeOut); + TStringOutput strOut(str); + TStringOutput checkOut(checkStr); + TBufferedOutput bufOut(&strOut, 8192); + TTeeOutput teeOut(&bufOut, &checkOut); + THttpOutput httpOut(&teeOut); - httpOut.EnableKeepAlive(true); - httpOut.EnableCompression(true); + httpOut.EnableKeepAlive(true); + httpOut.EnableCompression(true); - const char* header = "POST / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; - httpOut << header; + const char* header = "POST / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; + httpOut << header; UNIT_ASSERT_VALUES_EQUAL(str.size(), 0u); - const char* body = "<html>Hello</html>"; - httpOut << body; + const char* body = "<html>Hello</html>"; + httpOut << body; UNIT_ASSERT_VALUES_EQUAL(str.size(), 0u); - httpOut.Flush(); + httpOut.Flush(); UNIT_ASSERT_VALUES_EQUAL(checkStr.size(), str.size()); - } + } TString MakeHttpOutputBody(const char* body, bool encodingEnabled) { TString str; - TStringOutput strOut(str); - { - TBufferedOutput bufOut(&strOut, 8192); - THttpOutput httpOut(&bufOut); + TStringOutput strOut(str); + { + TBufferedOutput bufOut(&strOut, 8192); + THttpOutput httpOut(&bufOut); - httpOut.EnableKeepAlive(true); - httpOut.EnableCompression(true); + httpOut.EnableKeepAlive(true); + httpOut.EnableCompression(true); httpOut.EnableBodyEncoding(encodingEnabled); - httpOut << "POST / HTTP/1.1\r\n"; - httpOut << "Host: yandex.ru\r\n"; - httpOut << "Content-Encoding: gzip\r\n"; - httpOut << "\r\n"; + httpOut << "POST / HTTP/1.1\r\n"; + httpOut << "Host: yandex.ru\r\n"; + httpOut << "Content-Encoding: gzip\r\n"; + httpOut << "\r\n"; UNIT_ASSERT_VALUES_EQUAL(str.size(), 0u); - httpOut << body; - } - const char* bodyDelimiter = "\r\n\r\n"; - size_t bodyPos = str.find(bodyDelimiter); + httpOut << body; + } + const char* bodyDelimiter = "\r\n\r\n"; + size_t bodyPos = str.find(bodyDelimiter); UNIT_ASSERT(bodyPos != TString::npos); return str.substr(bodyPos + strlen(bodyDelimiter)); }; @@ -370,54 +370,54 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { const char* body = "<html>Hello</html>"; UNIT_ASSERT(MakeHttpOutputBody(body, false) == body); UNIT_ASSERT(MakeHttpOutputBody(body, true) == SimulateBodyEncoding(body)); - } - + } + Y_UNIT_TEST(TestOutputFinish) { TString str; - TStringOutput strOut(str); - TBufferedOutput bufOut(&strOut, 8192); + TStringOutput strOut(str); + TBufferedOutput bufOut(&strOut, 8192); THttpOutput httpOut(&bufOut); httpOut.EnableKeepAlive(true); httpOut.EnableCompression(true); - const char* header = "GET / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; - httpOut << header; + const char* header = "GET / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n"; + httpOut << header; unsigned curLen = str.size(); - const char* body = "<html>Hello</html>"; + const char* body = "<html>Hello</html>"; httpOut << body; UNIT_ASSERT_VALUES_EQUAL(curLen, str.size()); - httpOut.Finish(); + httpOut.Finish(); UNIT_ASSERT_VALUES_EQUAL(curLen + strlen(body), str.size()); } Y_UNIT_TEST(TestMultilineHeaders) { - const char* headerLine0 = "HTTP/1.1 200 OK"; - const char* headerLine1 = "Content-Language: en"; - const char* headerLine2 = "Vary: Accept-Encoding, "; - const char* headerLine3 = "\tAccept-Language"; - const char* headerLine4 = "Content-Length: 18"; + const char* headerLine0 = "HTTP/1.1 200 OK"; + const char* headerLine1 = "Content-Language: en"; + const char* headerLine2 = "Vary: Accept-Encoding, "; + const char* headerLine3 = "\tAccept-Language"; + const char* headerLine4 = "Content-Length: 18"; TString endLine("\r\n"); TString r; - r += headerLine0 + endLine; - r += headerLine1 + endLine; - r += headerLine2 + endLine; - r += headerLine3 + endLine; - r += headerLine4 + endLine + endLine; - r += "<html>Hello</html>"; - TStringInput stringInput(r); - THttpInput input(&stringInput); - - const THttpHeaders& httpHeaders = input.Headers(); - UNIT_ASSERT_VALUES_EQUAL(httpHeaders.Count(), 3u); - - THttpHeaders::TConstIterator it = httpHeaders.Begin(); + r += headerLine0 + endLine; + r += headerLine1 + endLine; + r += headerLine2 + endLine; + r += headerLine3 + endLine; + r += headerLine4 + endLine + endLine; + r += "<html>Hello</html>"; + TStringInput stringInput(r); + THttpInput input(&stringInput); + + const THttpHeaders& httpHeaders = input.Headers(); + UNIT_ASSERT_VALUES_EQUAL(httpHeaders.Count(), 3u); + + THttpHeaders::TConstIterator it = httpHeaders.Begin(); UNIT_ASSERT_VALUES_EQUAL(it->ToString(), TString(headerLine1)); UNIT_ASSERT_VALUES_EQUAL((++it)->ToString(), TString::Join(headerLine2, headerLine3)); UNIT_ASSERT_VALUES_EQUAL((++it)->ToString(), TString(headerLine4)); - } + } Y_UNIT_TEST(ContentLengthRemoval) { TMemoryInput request("GET / HTTP/1.1\r\nAccept-Encoding: gzip\r\n\r\n"); @@ -430,8 +430,8 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { httpOut.EnableCompression(true); httpOut << "HTTP/1.1 200 OK\r\n"; char answer[] = "Mary had a little lamb."; - httpOut << "Content-Length: " << strlen(answer) << "\r\n" - "\r\n"; + httpOut << "Content-Length: " << strlen(answer) << "\r\n" + "\r\n"; httpOut << answer; httpOut.Finish(); @@ -541,9 +541,9 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { TStringStream request; { THttpOutput httpOutput(&request); - httpOutput << "POST / HTTP/1.1\r\n" - "Host: yandex.ru\r\n" - "\r\n"; + httpOutput << "POST / HTTP/1.1\r\n" + "Host: yandex.ru\r\n" + "\r\n"; httpOutput << "GGLOL"; } { @@ -568,9 +568,9 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { Y_UNIT_TEST(TestInputHasContent) { { TStringStream request; - request << "POST / HTTP/1.1\r\n" - "Host: yandex.ru\r\n" - "\r\n"; + request << "POST / HTTP/1.1\r\n" + "Host: yandex.ru\r\n" + "\r\n"; request << "HTTPDATA"; TStringInput input(request.Str()); @@ -582,10 +582,10 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { { TStringStream request; - request << "POST / HTTP/1.1\r\n" - "Host: yandex.ru\r\n" - "Content-Length: 8" - "\r\n\r\n"; + request << "POST / HTTP/1.1\r\n" + "Host: yandex.ru\r\n" + "Content-Length: 8" + "\r\n\r\n"; request << "HTTPDATA"; TStringInput input(request.Str()); @@ -597,10 +597,10 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { { TStringStream request; - request << "POST / HTTP/1.1\r\n" - "Host: yandex.ru\r\n" - "Transfer-Encoding: chunked" - "\r\n\r\n"; + request << "POST / HTTP/1.1\r\n" + "Host: yandex.ru\r\n" + "Transfer-Encoding: chunked" + "\r\n\r\n"; request << "8\r\nHTTPDATA\r\n0\r\n"; TStringInput input(request.Str()); @@ -612,10 +612,10 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { } Y_UNIT_TEST(TestHttpInputHeadRequest) { - class THeadOnlyInput: public IInputStream { + class THeadOnlyInput: public IInputStream { public: THeadOnlyInput() = default; - + private: size_t DoRead(void* buf, size_t len) override { if (Eof_) { @@ -635,8 +635,8 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { private: TString Data_{TStringBuf("HEAD / HTTP/1.1\r\nHost: yandex.ru\r\n\r\n")}; - size_t Pos_{0}; - bool Eof_{false}; + size_t Pos_{0}; + bool Eof_{false}; }; THeadOnlyInput input; THttpInput httpInput(&input); @@ -647,10 +647,10 @@ Y_UNIT_TEST_SUITE(THttpStreamTest) { Y_UNIT_TEST(TestHttpOutputResponseToHeadRequestNoZeroChunk) { TStringStream request; - request << "HEAD / HTTP/1.1\r\n" - "Host: yandex.ru\r\n" - "Connection: Keep-Alive\r\n" - "\r\n"; + request << "HEAD / HTTP/1.1\r\n" + "Host: yandex.ru\r\n" + "Connection: Keep-Alive\r\n" + "\r\n"; TStringInput input(request.Str()); THttpInput httpInput(&input); diff --git a/library/cpp/http/io/stream_ut_medium.cpp b/library/cpp/http/io/stream_ut_medium.cpp index 2c125eb21e..e70d47555d 100644 --- a/library/cpp/http/io/stream_ut_medium.cpp +++ b/library/cpp/http/io/stream_ut_medium.cpp @@ -1,54 +1,54 @@ -#include "stream.h" +#include "stream.h" #include <library/cpp/testing/unittest/registar.h> -#include <util/stream/zlib.h> - +#include <util/stream/zlib.h> + Y_UNIT_TEST_SUITE(THttpTestMedium) { Y_UNIT_TEST(TestCodings2) { - TStringBuf data = "aaaaaaaaaaaaaaaaaaaaaaa"; - - for (auto codec : SupportedCodings()) { + TStringBuf data = "aaaaaaaaaaaaaaaaaaaaaaa"; + + for (auto codec : SupportedCodings()) { if (codec == TStringBuf("z-zlib-0")) { - continue; - } - + continue; + } + if (codec == TStringBuf("z-null")) { - continue; - } - - TString s; - - { - TStringOutput so(s); - THttpOutput ho(&so); - TBufferedOutput bo(&ho, 10000); - - bo << "HTTP/1.1 200 Ok\r\n" - << "Connection: close\r\n" - << "Content-Encoding: " << codec << "\r\n\r\n"; - - for (size_t i = 0; i < 100; ++i) { - bo << data; - } - } - - try { + continue; + } + + TString s; + + { + TStringOutput so(s); + THttpOutput ho(&so); + TBufferedOutput bo(&ho, 10000); + + bo << "HTTP/1.1 200 Ok\r\n" + << "Connection: close\r\n" + << "Content-Encoding: " << codec << "\r\n\r\n"; + + for (size_t i = 0; i < 100; ++i) { + bo << data; + } + } + + try { UNIT_ASSERT(s.size() > 10); - UNIT_ASSERT(s.find(data) == TString::npos); - } catch (...) { - Cerr << codec << " " << s << Endl; - - throw; - } - - { - TStringInput si(s); - THttpInput hi(&si); - - auto res = hi.ReadAll(); - - UNIT_ASSERT(res.find(data) == 0); - } - } - } - + UNIT_ASSERT(s.find(data) == TString::npos); + } catch (...) { + Cerr << codec << " " << s << Endl; + + throw; + } + + { + TStringInput si(s); + THttpInput hi(&si); + + auto res = hi.ReadAll(); + + UNIT_ASSERT(res.find(data) == 0); + } + } + } + } // THttpTestMedium suite diff --git a/library/cpp/http/io/ut/medium/ya.make b/library/cpp/http/io/ut/medium/ya.make index 235a23dcd7..24d16b00ea 100644 --- a/library/cpp/http/io/ut/medium/ya.make +++ b/library/cpp/http/io/ut/medium/ya.make @@ -1,11 +1,11 @@ UNITTEST_FOR(library/cpp/http/io) - + SIZE(MEDIUM) OWNER(g:util) - -SRCS( + +SRCS( stream_ut_medium.cpp -) - -END() +) + +END() diff --git a/library/cpp/http/io/ut/ya.make b/library/cpp/http/io/ut/ya.make index 84f6949db3..ea0597ebb6 100644 --- a/library/cpp/http/io/ut/ya.make +++ b/library/cpp/http/io/ut/ya.make @@ -1,16 +1,16 @@ UNITTEST_FOR(library/cpp/http/io) - + OWNER(g:util) - + PEERDIR( library/cpp/http/server ) -SRCS( - chunk_ut.cpp +SRCS( + chunk_ut.cpp compression_ut.cpp headers_ut.cpp - stream_ut.cpp -) - -END() + stream_ut.cpp +) + +END() diff --git a/library/cpp/http/io/ya.make b/library/cpp/http/io/ya.make index dcfbd79885..83b30c8958 100644 --- a/library/cpp/http/io/ya.make +++ b/library/cpp/http/io/ya.make @@ -1,22 +1,22 @@ -LIBRARY() - +LIBRARY() + OWNER( g:util mvel ) - -PEERDIR( + +PEERDIR( library/cpp/blockcodecs library/cpp/streams/brotli library/cpp/streams/bzip2 library/cpp/streams/lzma -) - -SRCS( - chunk.cpp +) + +SRCS( + chunk.cpp compression.cpp - headers.cpp + headers.cpp stream.cpp -) - -END() +) + +END() diff --git a/library/cpp/http/misc/httpcodes.cpp b/library/cpp/http/misc/httpcodes.cpp index ad8c80ac1e..fb9e43c9d3 100644 --- a/library/cpp/http/misc/httpcodes.cpp +++ b/library/cpp/http/misc/httpcodes.cpp @@ -2,26 +2,26 @@ TStringBuf HttpCodeStrEx(int code) noexcept { switch (code) { - case HTTP_CONTINUE: + case HTTP_CONTINUE: return TStringBuf("100 Continue"); - case HTTP_SWITCHING_PROTOCOLS: + case HTTP_SWITCHING_PROTOCOLS: return TStringBuf("101 Switching protocols"); case HTTP_PROCESSING: return TStringBuf("102 Processing"); - - case HTTP_OK: + + case HTTP_OK: return TStringBuf("200 Ok"); - case HTTP_CREATED: + case HTTP_CREATED: return TStringBuf("201 Created"); - case HTTP_ACCEPTED: + case HTTP_ACCEPTED: return TStringBuf("202 Accepted"); - case HTTP_NON_AUTHORITATIVE_INFORMATION: + case HTTP_NON_AUTHORITATIVE_INFORMATION: return TStringBuf("203 None authoritative information"); - case HTTP_NO_CONTENT: + case HTTP_NO_CONTENT: return TStringBuf("204 No content"); - case HTTP_RESET_CONTENT: + case HTTP_RESET_CONTENT: return TStringBuf("205 Reset content"); - case HTTP_PARTIAL_CONTENT: + case HTTP_PARTIAL_CONTENT: return TStringBuf("206 Partial content"); case HTTP_MULTI_STATUS: return TStringBuf("207 Multi status"); @@ -30,58 +30,58 @@ TStringBuf HttpCodeStrEx(int code) noexcept { case HTTP_IM_USED: return TStringBuf("226 IM used"); - case HTTP_MULTIPLE_CHOICES: + case HTTP_MULTIPLE_CHOICES: return TStringBuf("300 Multiple choices"); - case HTTP_MOVED_PERMANENTLY: + case HTTP_MOVED_PERMANENTLY: return TStringBuf("301 Moved permanently"); - case HTTP_FOUND: + case HTTP_FOUND: return TStringBuf("302 Moved temporarily"); - case HTTP_SEE_OTHER: + case HTTP_SEE_OTHER: return TStringBuf("303 See other"); - case HTTP_NOT_MODIFIED: + case HTTP_NOT_MODIFIED: return TStringBuf("304 Not modified"); - case HTTP_USE_PROXY: + case HTTP_USE_PROXY: return TStringBuf("305 Use proxy"); - case HTTP_TEMPORARY_REDIRECT: + case HTTP_TEMPORARY_REDIRECT: return TStringBuf("307 Temporarily redirect"); case HTTP_PERMANENT_REDIRECT: return TStringBuf("308 Permanent redirect"); - case HTTP_BAD_REQUEST: + case HTTP_BAD_REQUEST: return TStringBuf("400 Bad request"); - case HTTP_UNAUTHORIZED: + case HTTP_UNAUTHORIZED: return TStringBuf("401 Unauthorized"); - case HTTP_PAYMENT_REQUIRED: + case HTTP_PAYMENT_REQUIRED: return TStringBuf("402 Payment required"); - case HTTP_FORBIDDEN: + case HTTP_FORBIDDEN: return TStringBuf("403 Forbidden"); - case HTTP_NOT_FOUND: + case HTTP_NOT_FOUND: return TStringBuf("404 Not found"); - case HTTP_METHOD_NOT_ALLOWED: + case HTTP_METHOD_NOT_ALLOWED: return TStringBuf("405 Method not allowed"); - case HTTP_NOT_ACCEPTABLE: + case HTTP_NOT_ACCEPTABLE: return TStringBuf("406 Not acceptable"); - case HTTP_PROXY_AUTHENTICATION_REQUIRED: + case HTTP_PROXY_AUTHENTICATION_REQUIRED: return TStringBuf("407 Proxy Authentication required"); - case HTTP_REQUEST_TIME_OUT: + case HTTP_REQUEST_TIME_OUT: return TStringBuf("408 Request time out"); - case HTTP_CONFLICT: + case HTTP_CONFLICT: return TStringBuf("409 Conflict"); - case HTTP_GONE: + case HTTP_GONE: return TStringBuf("410 Gone"); - case HTTP_LENGTH_REQUIRED: + case HTTP_LENGTH_REQUIRED: return TStringBuf("411 Length required"); - case HTTP_PRECONDITION_FAILED: + case HTTP_PRECONDITION_FAILED: return TStringBuf("412 Precondition failed"); - case HTTP_REQUEST_ENTITY_TOO_LARGE: + case HTTP_REQUEST_ENTITY_TOO_LARGE: return TStringBuf("413 Request entity too large"); - case HTTP_REQUEST_URI_TOO_LARGE: + case HTTP_REQUEST_URI_TOO_LARGE: return TStringBuf("414 Request uri too large"); - case HTTP_UNSUPPORTED_MEDIA_TYPE: + case HTTP_UNSUPPORTED_MEDIA_TYPE: return TStringBuf("415 Unsupported media type"); - case HTTP_REQUESTED_RANGE_NOT_SATISFIABLE: + case HTTP_REQUESTED_RANGE_NOT_SATISFIABLE: return TStringBuf("416 Requested Range Not Satisfiable"); - case HTTP_EXPECTATION_FAILED: + case HTTP_EXPECTATION_FAILED: return TStringBuf("417 Expectation Failed"); case HTTP_I_AM_A_TEAPOT: return TStringBuf("418 I Am A Teapot"); @@ -89,7 +89,7 @@ TStringBuf HttpCodeStrEx(int code) noexcept { return TStringBuf("419 Authentication Timeout"); case HTTP_MISDIRECTED_REQUEST: return TStringBuf("421 Misdirected Request"); - case HTTP_UNPROCESSABLE_ENTITY: + case HTTP_UNPROCESSABLE_ENTITY: return TStringBuf("422 Unprocessable Entity"); case HTTP_LOCKED: return TStringBuf("423 Locked"); @@ -101,24 +101,24 @@ TStringBuf HttpCodeStrEx(int code) noexcept { return TStringBuf("426 Upgrade Required"); case HTTP_PRECONDITION_REQUIRED: return TStringBuf("428 Precondition Required"); - case HTTP_TOO_MANY_REQUESTS: + case HTTP_TOO_MANY_REQUESTS: return TStringBuf("429 Too Many Requests"); case HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE: return TStringBuf("431 Request Header Fields Too Large"); case HTTP_UNAVAILABLE_FOR_LEGAL_REASONS: return TStringBuf("451 Unavailable For Legal Reason"); - case HTTP_INTERNAL_SERVER_ERROR: + case HTTP_INTERNAL_SERVER_ERROR: return TStringBuf("500 Internal server error"); - case HTTP_NOT_IMPLEMENTED: + case HTTP_NOT_IMPLEMENTED: return TStringBuf("501 Not implemented"); - case HTTP_BAD_GATEWAY: + case HTTP_BAD_GATEWAY: return TStringBuf("502 Bad gateway"); - case HTTP_SERVICE_UNAVAILABLE: + case HTTP_SERVICE_UNAVAILABLE: return TStringBuf("503 Service unavailable"); - case HTTP_GATEWAY_TIME_OUT: + case HTTP_GATEWAY_TIME_OUT: return TStringBuf("504 Gateway time out"); - case HTTP_HTTP_VERSION_NOT_SUPPORTED: + case HTTP_HTTP_VERSION_NOT_SUPPORTED: return TStringBuf("505 HTTP version not supported"); case HTTP_VARIANT_ALSO_NEGOTIATES: return TStringBuf("506 Variant also negotiates"); @@ -135,7 +135,7 @@ TStringBuf HttpCodeStrEx(int code) noexcept { case HTTP_UNASSIGNED_512: return TStringBuf("512 Unassigned"); - default: + default: return TStringBuf("000 Unknown HTTP code"); } } diff --git a/library/cpp/http/misc/httpcodes.h b/library/cpp/http/misc/httpcodes.h index cbfbaa1188..2b7a3c4a80 100644 --- a/library/cpp/http/misc/httpcodes.h +++ b/library/cpp/http/misc/httpcodes.h @@ -3,67 +3,67 @@ #include <util/generic/strbuf.h> enum HttpCodes { - HTTP_CONTINUE = 100, - HTTP_SWITCHING_PROTOCOLS = 101, + HTTP_CONTINUE = 100, + HTTP_SWITCHING_PROTOCOLS = 101, HTTP_PROCESSING = 102, - HTTP_OK = 200, - HTTP_CREATED = 201, - HTTP_ACCEPTED = 202, - HTTP_NON_AUTHORITATIVE_INFORMATION = 203, - HTTP_NO_CONTENT = 204, - HTTP_RESET_CONTENT = 205, - HTTP_PARTIAL_CONTENT = 206, + HTTP_OK = 200, + HTTP_CREATED = 201, + HTTP_ACCEPTED = 202, + HTTP_NON_AUTHORITATIVE_INFORMATION = 203, + HTTP_NO_CONTENT = 204, + HTTP_RESET_CONTENT = 205, + HTTP_PARTIAL_CONTENT = 206, HTTP_MULTI_STATUS = 207, HTTP_ALREADY_REPORTED = 208, HTTP_IM_USED = 226, - HTTP_MULTIPLE_CHOICES = 300, - HTTP_MOVED_PERMANENTLY = 301, - HTTP_FOUND = 302, - HTTP_SEE_OTHER = 303, - HTTP_NOT_MODIFIED = 304, - HTTP_USE_PROXY = 305, - HTTP_TEMPORARY_REDIRECT = 307, + HTTP_MULTIPLE_CHOICES = 300, + HTTP_MOVED_PERMANENTLY = 301, + HTTP_FOUND = 302, + HTTP_SEE_OTHER = 303, + HTTP_NOT_MODIFIED = 304, + HTTP_USE_PROXY = 305, + HTTP_TEMPORARY_REDIRECT = 307, HTTP_PERMANENT_REDIRECT = 308, - HTTP_BAD_REQUEST = 400, - HTTP_UNAUTHORIZED = 401, - HTTP_PAYMENT_REQUIRED = 402, - HTTP_FORBIDDEN = 403, - HTTP_NOT_FOUND = 404, - HTTP_METHOD_NOT_ALLOWED = 405, - HTTP_NOT_ACCEPTABLE = 406, - HTTP_PROXY_AUTHENTICATION_REQUIRED = 407, - HTTP_REQUEST_TIME_OUT = 408, - HTTP_CONFLICT = 409, - HTTP_GONE = 410, - HTTP_LENGTH_REQUIRED = 411, - HTTP_PRECONDITION_FAILED = 412, - HTTP_REQUEST_ENTITY_TOO_LARGE = 413, - HTTP_REQUEST_URI_TOO_LARGE = 414, - HTTP_UNSUPPORTED_MEDIA_TYPE = 415, - HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416, - HTTP_EXPECTATION_FAILED = 417, + HTTP_BAD_REQUEST = 400, + HTTP_UNAUTHORIZED = 401, + HTTP_PAYMENT_REQUIRED = 402, + HTTP_FORBIDDEN = 403, + HTTP_NOT_FOUND = 404, + HTTP_METHOD_NOT_ALLOWED = 405, + HTTP_NOT_ACCEPTABLE = 406, + HTTP_PROXY_AUTHENTICATION_REQUIRED = 407, + HTTP_REQUEST_TIME_OUT = 408, + HTTP_CONFLICT = 409, + HTTP_GONE = 410, + HTTP_LENGTH_REQUIRED = 411, + HTTP_PRECONDITION_FAILED = 412, + HTTP_REQUEST_ENTITY_TOO_LARGE = 413, + HTTP_REQUEST_URI_TOO_LARGE = 414, + HTTP_UNSUPPORTED_MEDIA_TYPE = 415, + HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416, + HTTP_EXPECTATION_FAILED = 417, HTTP_I_AM_A_TEAPOT = 418, HTTP_AUTHENTICATION_TIMEOUT = 419, HTTP_MISDIRECTED_REQUEST = 421, - HTTP_UNPROCESSABLE_ENTITY = 422, + HTTP_UNPROCESSABLE_ENTITY = 422, HTTP_LOCKED = 423, HTTP_FAILED_DEPENDENCY = 424, HTTP_UNORDERED_COLLECTION = 425, HTTP_UPGRADE_REQUIRED = 426, HTTP_PRECONDITION_REQUIRED = 428, - HTTP_TOO_MANY_REQUESTS = 429, + HTTP_TOO_MANY_REQUESTS = 429, HTTP_REQUEST_HEADER_FIELDS_TOO_LARGE = 431, HTTP_UNAVAILABLE_FOR_LEGAL_REASONS = 451, - HTTP_INTERNAL_SERVER_ERROR = 500, - HTTP_NOT_IMPLEMENTED = 501, - HTTP_BAD_GATEWAY = 502, - HTTP_SERVICE_UNAVAILABLE = 503, - HTTP_GATEWAY_TIME_OUT = 504, - HTTP_HTTP_VERSION_NOT_SUPPORTED = 505, + HTTP_INTERNAL_SERVER_ERROR = 500, + HTTP_NOT_IMPLEMENTED = 501, + HTTP_BAD_GATEWAY = 502, + HTTP_SERVICE_UNAVAILABLE = 503, + HTTP_GATEWAY_TIME_OUT = 504, + HTTP_HTTP_VERSION_NOT_SUPPORTED = 505, HTTP_VARIANT_ALSO_NEGOTIATES = 506, HTTP_INSUFFICIENT_STORAGE = 507, HTTP_LOOP_DETECTED = 508, diff --git a/library/cpp/http/misc/httpdate.cpp b/library/cpp/http/misc/httpdate.cpp index 4a3031bbf4..98876293c6 100644 --- a/library/cpp/http/misc/httpdate.cpp +++ b/library/cpp/http/misc/httpdate.cpp @@ -43,7 +43,7 @@ static const char *wkdays[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; - + static const char *months[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" @@ -64,15 +64,15 @@ int format_http_date(char buf[], size_t size, time_t when) { #endif } -char* format_http_date(time_t when, char* buf, size_t buflen) { - const int len = format_http_date(buf, buflen, when); - - if (len == 0) { +char* format_http_date(time_t when, char* buf, size_t buflen) { + const int len = format_http_date(buf, buflen, when); + + if (len == 0) { return nullptr; - } - + } + Y_ASSERT(len > 0 && size_t(len) < buflen); - + return buf; } diff --git a/library/cpp/http/misc/httpdate.h b/library/cpp/http/misc/httpdate.h index 04876f38fe..0640624661 100644 --- a/library/cpp/http/misc/httpdate.h +++ b/library/cpp/http/misc/httpdate.h @@ -1,11 +1,11 @@ #pragma once -#include <util/datetime/base.h> +#include <util/datetime/base.h> #include <util/generic/string.h> - + #include <ctime> -#define BAD_DATE ((time_t)-1) +#define BAD_DATE ((time_t)-1) inline time_t parse_http_date(const TStringBuf& datestring) { try { @@ -16,6 +16,6 @@ inline time_t parse_http_date(const TStringBuf& datestring) { } int format_http_date(char buf[], size_t size, time_t when); -char* format_http_date(time_t when, char* buf, size_t len); +char* format_http_date(time_t when, char* buf, size_t len); TString FormatHttpDate(time_t when); diff --git a/library/cpp/http/misc/httpdate_ut.cpp b/library/cpp/http/misc/httpdate_ut.cpp index c1a0103501..ac4e54063e 100644 --- a/library/cpp/http/misc/httpdate_ut.cpp +++ b/library/cpp/http/misc/httpdate_ut.cpp @@ -1,15 +1,15 @@ #include <library/cpp/testing/unittest/registar.h> - -#include "httpdate.h" - + +#include "httpdate.h" + Y_UNIT_TEST_SUITE(TestHttpDate) { Y_UNIT_TEST(Test1) { - char buf1[100]; - char buf2[100]; - - UNIT_ASSERT((int)strlen(format_http_date(0, buf1, sizeof(buf1))) == format_http_date(buf2, sizeof(buf2), 0)); - } + char buf1[100]; + char buf2[100]; + + UNIT_ASSERT((int)strlen(format_http_date(0, buf1, sizeof(buf1))) == format_http_date(buf2, sizeof(buf2), 0)); + } Y_UNIT_TEST(Test2) { UNIT_ASSERT_STRINGS_EQUAL(FormatHttpDate(1234567890), "Fri, 13 Feb 2009 23:31:30 GMT"); } -} +} diff --git a/library/cpp/http/misc/httpreqdata.cpp b/library/cpp/http/misc/httpreqdata.cpp index f6951f68cd..ecc55e754d 100644 --- a/library/cpp/http/misc/httpreqdata.cpp +++ b/library/cpp/http/misc/httpreqdata.cpp @@ -1,51 +1,51 @@ -#include "httpreqdata.h" - +#include "httpreqdata.h" + #include <util/stream/mem.h> TBaseServerRequestData::TBaseServerRequestData(SOCKET s) : Addr(nullptr) - , Host() - , Port() + , Host() + , Port() , Path(nullptr) , Search(nullptr) - , SearchLength(0) - , Socket(s) - , BeginTime(MicroSeconds()) -{ -} - + , SearchLength(0) + , Socket(s) + , BeginTime(MicroSeconds()) +{ +} + TBaseServerRequestData::TBaseServerRequestData(const char* qs, SOCKET s) : Addr(nullptr) - , Host() - , Port() + , Host() + , Port() , Path(nullptr) - , Search((char*)qs) - , SearchLength(qs ? strlen(qs) : 0) + , Search((char*)qs) + , SearchLength(qs ? strlen(qs) : 0) , OrigSearch(Search, SearchLength) - , Socket(s) - , BeginTime(MicroSeconds()) -{ -} - + , Socket(s) + , BeginTime(MicroSeconds()) +{ +} + void TBaseServerRequestData::AppendQueryString(const char* str, size_t length) { - if (Y_UNLIKELY(Search)) { + if (Y_UNLIKELY(Search)) { Y_ASSERT(strlen(Search) == SearchLength); ModifiedQueryString.Reserve(SearchLength + length + 2); ModifiedQueryString.Assign(Search, SearchLength); - if (SearchLength > 0 && Search[SearchLength - 1] != '&' && - length > 0 && str[0] != '&') { + if (SearchLength > 0 && Search[SearchLength - 1] != '&' && + length > 0 && str[0] != '&') { ModifiedQueryString.Append('&'); - } + } ModifiedQueryString.Append(str, length); - } else { + } else { ModifiedQueryString.Reserve(length + 1); ModifiedQueryString.Assign(str, length); - } + } ModifiedQueryString.Append('\0'); Search = ModifiedQueryString.data(); SearchLength = ModifiedQueryString.size() - 1; // ignore terminator -} - +} + void TBaseServerRequestData::SetRemoteAddr(TStringBuf addr) { TMemoryOutput out(AddrData, Y_ARRAY_SIZE(AddrData) - 1); out.Write(addr.substr(0, Y_ARRAY_SIZE(AddrData) - 1)); @@ -55,142 +55,142 @@ void TBaseServerRequestData::SetRemoteAddr(TStringBuf addr) { } const char* TBaseServerRequestData::RemoteAddr() const { - if (!Addr) { - *AddrData = 0; - GetRemoteAddr(Socket, AddrData, sizeof(AddrData)); - Addr = AddrData; - } - - return Addr; -} - + if (!Addr) { + *AddrData = 0; + GetRemoteAddr(Socket, AddrData, sizeof(AddrData)); + Addr = AddrData; + } + + return Addr; +} + const char* TBaseServerRequestData::HeaderIn(TStringBuf key) const { auto it = HeadersIn_.find(key); - + if (it == HeadersIn_.end()) { return nullptr; - } - + } + return it->second.data(); -} - +} + TString TBaseServerRequestData::HeaderByIndex(size_t n) const noexcept { - if (n >= HeadersCount()) { + if (n >= HeadersCount()) { return nullptr; - } - + } + THttpHeadersContainer::const_iterator i = HeadersIn_.begin(); - - while (n) { - ++i; - --n; - } - + + while (n) { + ++i; + --n; + } + return TString(i->first) + TStringBuf(": ") + i->second; -} - +} + const char* TBaseServerRequestData::Environment(const char* key) const { - if (stricmp(key, "REMOTE_ADDR") == 0) { - const char* ip = HeaderIn("X-Real-IP"); - if (ip) - return ip; - return RemoteAddr(); - } else if (stricmp(key, "QUERY_STRING") == 0) { - return QueryString(); - } else if (stricmp(key, "SERVER_NAME") == 0) { + if (stricmp(key, "REMOTE_ADDR") == 0) { + const char* ip = HeaderIn("X-Real-IP"); + if (ip) + return ip; + return RemoteAddr(); + } else if (stricmp(key, "QUERY_STRING") == 0) { + return QueryString(); + } else if (stricmp(key, "SERVER_NAME") == 0) { return ServerName().data(); - } else if (stricmp(key, "SERVER_PORT") == 0) { + } else if (stricmp(key, "SERVER_PORT") == 0) { return ServerPort().data(); - } else if (stricmp(key, "SCRIPT_NAME") == 0) { - return ScriptName(); - } + } else if (stricmp(key, "SCRIPT_NAME") == 0) { + return ScriptName(); + } return nullptr; -} - +} + void TBaseServerRequestData::Clear() { HeadersIn_.clear(); Addr = Path = Search = nullptr; OrigSearch = {}; - SearchLength = 0; - Host.clear(); - Port.clear(); - CurPage.remove(); + SearchLength = 0; + Host.clear(); + Port.clear(); + CurPage.remove(); ParseBuf.Clear(); - BeginTime = MicroSeconds(); -} - + BeginTime = MicroSeconds(); +} + const char* TBaseServerRequestData::GetCurPage() const { - if (!CurPage && Host) { - CurPage = "http://"; - CurPage += Host; - if (Port) { - CurPage += ':'; - CurPage += Port; - } - CurPage += Path; - if (Search) { - CurPage += '?'; - CurPage += Search; - } - } + if (!CurPage && Host) { + CurPage = "http://"; + CurPage += Host; + if (Port) { + CurPage += ':'; + CurPage += Port; + } + CurPage += Path; + if (Search) { + CurPage += '?'; + CurPage += Search; + } + } return CurPage.data(); -} - +} + bool TBaseServerRequestData::Parse(const char* origReq) { - size_t origReqLength = strlen(origReq); + size_t origReqLength = strlen(origReq); ParseBuf.Assign(origReq, origReqLength + 1); char* req = ParseBuf.Data(); - - while (*req == ' ' || *req == '\t') - req++; - if (*req != '/') - return false; // we are not a proxy - while (req[1] == '/') // remove redundant slashes - req++; - - // detect url end (can contain some garbage after whitespace, e.g. 'HTTP 1.1') - char* urlEnd = req; - while (*urlEnd && *urlEnd != ' ' && *urlEnd != '\t') - urlEnd++; - if (*urlEnd) - *urlEnd = 0; - - // cut fragment if exists - char* fragment = strchr(req, '#'); - if (fragment) - *fragment = 0; // ignore fragment - else - fragment = urlEnd; - Path = req; - - // calculate Search length without additional strlen-ing - Search = strchr(Path, '?'); - if (Search) { - *Search++ = 0; - ptrdiff_t delta = fragment - Search; - // indeed, second case is a parse error - SearchLength = (delta >= 0) ? delta : (urlEnd - Search); + + while (*req == ' ' || *req == '\t') + req++; + if (*req != '/') + return false; // we are not a proxy + while (req[1] == '/') // remove redundant slashes + req++; + + // detect url end (can contain some garbage after whitespace, e.g. 'HTTP 1.1') + char* urlEnd = req; + while (*urlEnd && *urlEnd != ' ' && *urlEnd != '\t') + urlEnd++; + if (*urlEnd) + *urlEnd = 0; + + // cut fragment if exists + char* fragment = strchr(req, '#'); + if (fragment) + *fragment = 0; // ignore fragment + else + fragment = urlEnd; + Path = req; + + // calculate Search length without additional strlen-ing + Search = strchr(Path, '?'); + if (Search) { + *Search++ = 0; + ptrdiff_t delta = fragment - Search; + // indeed, second case is a parse error + SearchLength = (delta >= 0) ? delta : (urlEnd - Search); Y_ASSERT(strlen(Search) == SearchLength); - } else { - SearchLength = 0; - } + } else { + SearchLength = 0; + } OrigSearch = {Search, SearchLength}; - return true; -} - + return true; +} + void TBaseServerRequestData::AddHeader(const TString& name, const TString& value) { HeadersIn_[name] = value; - + if (stricmp(name.data(), "Host") == 0) { size_t hostLen = strcspn(value.data(), ":"); if (value[hostLen] == ':') Port = value.substr(hostLen + 1); Host = value.substr(0, hostLen); - } -} - + } +} + void TBaseServerRequestData::SetPath(const TString& path) { PathStorage = TBuffer(path.data(), path.size() + 1); Path = PathStorage.Data(); -} +} diff --git a/library/cpp/http/misc/httpreqdata.h b/library/cpp/http/misc/httpreqdata.h index 16e59c4d78..946a0e5f14 100644 --- a/library/cpp/http/misc/httpreqdata.h +++ b/library/cpp/http/misc/httpreqdata.h @@ -1,14 +1,14 @@ #pragma once #include <library/cpp/digest/lower_case/hash_ops.h> - -#include <util/str_stl.h> - + +#include <util/str_stl.h> + #include <util/system/defaults.h> -#include <util/string/cast.h> +#include <util/string/cast.h> #include <library/cpp/cgiparam/cgiparam.h> #include <util/network/address.h> -#include <util/network/socket.h> +#include <util/network/socket.h> #include <util/generic/hash.h> #include <util/system/yassert.h> #include <util/generic/string.h> @@ -21,16 +21,16 @@ class TBaseServerRequestData { public: TBaseServerRequestData(SOCKET s = INVALID_SOCKET); TBaseServerRequestData(const char* qs, SOCKET s = INVALID_SOCKET); - + void SetHost(const TString& host, ui16 port) { Host = host; Port = ToString(port); } - + const TString& ServerName() const { return Host; } - + NAddr::IRemoteAddrPtr ServerAddress() const { return NAddr::GetSockAddr(Socket); } @@ -38,57 +38,57 @@ public: const TString& ServerPort() const { return Port; } - + const char* ScriptName() const { return Path; } - + const char* QueryString() const { return Search; } - + TStringBuf QueryStringBuf() const { return TStringBuf(Search, SearchLength); } - + TStringBuf OrigQueryStringBuf() const { return OrigSearch; } - void AppendQueryString(const char* str, size_t length); - const char* RemoteAddr() const; + void AppendQueryString(const char* str, size_t length); + const char* RemoteAddr() const; void SetRemoteAddr(TStringBuf addr); const char* HeaderIn(TStringBuf key) const; - + const THttpHeadersContainer& HeadersIn() const { return HeadersIn_; } inline size_t HeadersCount() const noexcept { return HeadersIn_.size(); - } - + } + TString HeaderByIndex(size_t n) const noexcept; - const char* Environment(const char* key) const; - - void Clear(); - + const char* Environment(const char* key) const; + + void Clear(); + void SetSocket(SOCKET s) noexcept { - Socket = s; - } - + Socket = s; + } + ui64 RequestBeginTime() const noexcept { - return BeginTime; - } - + return BeginTime; + } + void SetPath(const TString& path); const char* GetCurPage() const; bool Parse(const char* req); void AddHeader(const TString& name, const TString& value); - + private: TBuffer PathStorage; - mutable char* Addr; + mutable char* Addr; TString Host; TString Port; char* Path; @@ -98,13 +98,13 @@ private: THttpHeadersContainer HeadersIn_; mutable char AddrData[INET6_ADDRSTRLEN]; SOCKET Socket; - ui64 BeginTime; + ui64 BeginTime; mutable TString CurPage; TBuffer ParseBuf; TBuffer ModifiedQueryString; }; - -class TServerRequestData: public TBaseServerRequestData { + +class TServerRequestData: public TBaseServerRequestData { public: TServerRequestData(SOCKET s = INVALID_SOCKET) : TBaseServerRequestData(s) @@ -119,7 +119,7 @@ public: void Scan() { CgiParam.Scan(QueryStringBuf()); } - + public: - TCgiParameters CgiParam; + TCgiParameters CgiParam; }; diff --git a/library/cpp/http/misc/httpreqdata_ut.cpp b/library/cpp/http/misc/httpreqdata_ut.cpp index e7f16ef27c..24225e78ac 100644 --- a/library/cpp/http/misc/httpreqdata_ut.cpp +++ b/library/cpp/http/misc/httpreqdata_ut.cpp @@ -4,19 +4,19 @@ Y_UNIT_TEST_SUITE(TRequestServerDataTest) { Y_UNIT_TEST(Headers) { - TServerRequestData sd; - + TServerRequestData sd; + sd.AddHeader("x-xx", "y-yy"); sd.AddHeader("x-Xx", "y-yy"); - - UNIT_ASSERT_VALUES_EQUAL(sd.HeadersCount(), 1); - + + UNIT_ASSERT_VALUES_EQUAL(sd.HeadersCount(), 1); + sd.AddHeader("x-XxX", "y-yyy"); - UNIT_ASSERT_VALUES_EQUAL(sd.HeadersCount(), 2); + UNIT_ASSERT_VALUES_EQUAL(sd.HeadersCount(), 2); UNIT_ASSERT_VALUES_EQUAL(TStringBuf(sd.HeaderIn("X-XX")), TStringBuf("y-yy")); UNIT_ASSERT_VALUES_EQUAL(TStringBuf(sd.HeaderIn("X-XXX")), TStringBuf("y-yyy")); - } - + } + Y_UNIT_TEST(ComplexHeaders) { TServerRequestData sd; sd.SetHost("zzz", 1); @@ -43,24 +43,24 @@ Y_UNIT_TEST_SUITE(TRequestServerDataTest) { } Y_UNIT_TEST(ParseScan) { - TServerRequestData rd; + TServerRequestData rd; - // Parse parses url without host - UNIT_ASSERT(!rd.Parse(" http://yandex.ru/yandsearch?>a=fake&haha=da HTTP 1.1 OK")); + // Parse parses url without host + UNIT_ASSERT(!rd.Parse(" http://yandex.ru/yandsearch?>a=fake&haha=da HTTP 1.1 OK")); - // This should work - UNIT_ASSERT(rd.Parse(" /yandsearch?>a=fake&haha=da HTTP 1.1 OK")); + // This should work + UNIT_ASSERT(rd.Parse(" /yandsearch?>a=fake&haha=da HTTP 1.1 OK")); UNIT_ASSERT_STRINGS_EQUAL(rd.QueryStringBuf(), ">a=fake&haha=da"); UNIT_ASSERT_STRINGS_EQUAL(rd.QueryStringBuf(), rd.OrigQueryStringBuf()); - rd.Scan(); - UNIT_ASSERT(rd.CgiParam.Has("gta", "fake")); - UNIT_ASSERT(rd.CgiParam.Has("haha", "da")); - UNIT_ASSERT(!rd.CgiParam.Has("no-param")); + rd.Scan(); + UNIT_ASSERT(rd.CgiParam.Has("gta", "fake")); + UNIT_ASSERT(rd.CgiParam.Has("haha", "da")); + UNIT_ASSERT(!rd.CgiParam.Has("no-param")); - rd.Clear(); - } + rd.Clear(); + } Y_UNIT_TEST(Ctor) { const TString qs("gta=fake&haha=da"); @@ -69,55 +69,55 @@ Y_UNIT_TEST_SUITE(TRequestServerDataTest) { UNIT_ASSERT_STRINGS_EQUAL(rd.QueryStringBuf(), qs); UNIT_ASSERT_STRINGS_EQUAL(rd.OrigQueryStringBuf(), qs); - UNIT_ASSERT(rd.CgiParam.Has("gta")); - UNIT_ASSERT(rd.CgiParam.Has("haha")); - UNIT_ASSERT(!rd.CgiParam.Has("no-param")); - } + UNIT_ASSERT(rd.CgiParam.Has("gta")); + UNIT_ASSERT(rd.CgiParam.Has("haha")); + UNIT_ASSERT(!rd.CgiParam.Has("no-param")); + } Y_UNIT_TEST(HashCut) { const TString qs(">a=fake&haha=da"); const TString header = " /yandsearch?" + qs + "#&uberParam=yes&q=? HTTP 1.1 OK"; - TServerRequestData rd; + TServerRequestData rd; rd.Parse(header.c_str()); UNIT_ASSERT_STRINGS_EQUAL(rd.QueryStringBuf(), qs); UNIT_ASSERT_STRINGS_EQUAL(rd.OrigQueryStringBuf(), qs); - rd.Scan(); - UNIT_ASSERT(rd.CgiParam.Has("gta")); - UNIT_ASSERT(rd.CgiParam.Has("haha")); - UNIT_ASSERT(!rd.CgiParam.Has("uberParam")); - } + rd.Scan(); + UNIT_ASSERT(rd.CgiParam.Has("gta")); + UNIT_ASSERT(rd.CgiParam.Has("haha")); + UNIT_ASSERT(!rd.CgiParam.Has("uberParam")); + } Y_UNIT_TEST(MisplacedHashCut) { - TServerRequestData rd; - rd.Parse(" /y#ndsearch?>a=fake&haha=da&uberParam=yes&q=? HTTP 1.1 OK"); + TServerRequestData rd; + rd.Parse(" /y#ndsearch?>a=fake&haha=da&uberParam=yes&q=? HTTP 1.1 OK"); UNIT_ASSERT_STRINGS_EQUAL(rd.QueryStringBuf(), ""); UNIT_ASSERT_STRINGS_EQUAL(rd.OrigQueryStringBuf(), ""); - rd.Scan(); - UNIT_ASSERT(rd.CgiParam.empty()); - } + rd.Scan(); + UNIT_ASSERT(rd.CgiParam.empty()); + } Y_UNIT_TEST(CornerCase) { - TServerRequestData rd; - rd.Parse(" /yandsearch?#"); + TServerRequestData rd; + rd.Parse(" /yandsearch?#"); UNIT_ASSERT_STRINGS_EQUAL(rd.QueryStringBuf(), ""); UNIT_ASSERT_STRINGS_EQUAL(rd.OrigQueryStringBuf(), ""); - rd.Scan(); - UNIT_ASSERT(rd.CgiParam.empty()); - } + rd.Scan(); + UNIT_ASSERT(rd.CgiParam.empty()); + } Y_UNIT_TEST(AppendQueryString) { const TString qs("gta=fake&haha=da"); TServerRequestData rd(qs.c_str()); - UNIT_ASSERT(rd.CgiParam.Has("gta", "fake")); - UNIT_ASSERT(rd.CgiParam.Has("haha", "da")); + UNIT_ASSERT(rd.CgiParam.Has("gta", "fake")); + UNIT_ASSERT(rd.CgiParam.Has("haha", "da")); UNIT_ASSERT_STRINGS_EQUAL(rd.QueryStringBuf(), qs); UNIT_ASSERT_STRINGS_EQUAL(rd.QueryStringBuf(), rd.OrigQueryStringBuf()); @@ -128,11 +128,11 @@ Y_UNIT_TEST_SUITE(TRequestServerDataTest) { UNIT_ASSERT_STRINGS_EQUAL(rd.QueryStringBuf(), qs + '&' + appendix); UNIT_ASSERT_STRINGS_EQUAL(rd.OrigQueryStringBuf(), qs); - rd.Scan(); + rd.Scan(); - UNIT_ASSERT(rd.CgiParam.Has("gta", "true")); - UNIT_ASSERT(rd.CgiParam.Has("gta", "new")); - } + UNIT_ASSERT(rd.CgiParam.Has("gta", "true")); + UNIT_ASSERT(rd.CgiParam.Has("gta", "new")); + } Y_UNIT_TEST(SetRemoteAddrSimple) { static const TString TEST = "abacaba.search.yandex.net"; diff --git a/library/cpp/http/misc/parsed_request.cpp b/library/cpp/http/misc/parsed_request.cpp index e332a24e91..8568293e2c 100644 --- a/library/cpp/http/misc/parsed_request.cpp +++ b/library/cpp/http/misc/parsed_request.cpp @@ -1,32 +1,32 @@ -#include "parsed_request.h" - +#include "parsed_request.h" + #include <util/string/strip.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <util/string/cast.h> - + static inline TStringBuf StripLeft(const TStringBuf& s) noexcept { - const char* b = s.begin(); - const char* e = s.end(); - - StripRangeBegin(b, e); - - return TStringBuf(b, e); -} - -TParsedHttpRequest::TParsedHttpRequest(const TStringBuf& str) { - TStringBuf tmp; - + const char* b = s.begin(); + const char* e = s.end(); + + StripRangeBegin(b, e); + + return TStringBuf(b, e); +} + +TParsedHttpRequest::TParsedHttpRequest(const TStringBuf& str) { + TStringBuf tmp; + if (!StripLeft(str).TrySplit(' ', Method, tmp)) { - ythrow yexception() << "bad request(" << ToString(str).Quote() << ")"; - } - + ythrow yexception() << "bad request(" << ToString(str).Quote() << ")"; + } + if (!StripLeft(tmp).TrySplit(' ', Request, Proto)) { - ythrow yexception() << "bad request(" << ToString(str).Quote() << ")"; - } - - Proto = StripLeft(Proto); -} - -TParsedHttpLocation::TParsedHttpLocation(const TStringBuf& req) { - req.Split('?', Path, Cgi); -} + ythrow yexception() << "bad request(" << ToString(str).Quote() << ")"; + } + + Proto = StripLeft(Proto); +} + +TParsedHttpLocation::TParsedHttpLocation(const TStringBuf& req) { + req.Split('?', Path, Cgi); +} diff --git a/library/cpp/http/misc/parsed_request.h b/library/cpp/http/misc/parsed_request.h index d4df705495..00ba7767e7 100644 --- a/library/cpp/http/misc/parsed_request.h +++ b/library/cpp/http/misc/parsed_request.h @@ -1,26 +1,26 @@ #pragma once - -#include <util/generic/strbuf.h> - -struct TParsedHttpRequest { - TParsedHttpRequest(const TStringBuf& str); - - TStringBuf Method; - TStringBuf Request; - TStringBuf Proto; -}; - -struct TParsedHttpLocation { - TParsedHttpLocation(const TStringBuf& req); - - TStringBuf Path; - TStringBuf Cgi; -}; - -struct TParsedHttpFull: public TParsedHttpRequest, public TParsedHttpLocation { - inline TParsedHttpFull(const TStringBuf& line) - : TParsedHttpRequest(line) - , TParsedHttpLocation(Request) - { - } -}; + +#include <util/generic/strbuf.h> + +struct TParsedHttpRequest { + TParsedHttpRequest(const TStringBuf& str); + + TStringBuf Method; + TStringBuf Request; + TStringBuf Proto; +}; + +struct TParsedHttpLocation { + TParsedHttpLocation(const TStringBuf& req); + + TStringBuf Path; + TStringBuf Cgi; +}; + +struct TParsedHttpFull: public TParsedHttpRequest, public TParsedHttpLocation { + inline TParsedHttpFull(const TStringBuf& line) + : TParsedHttpRequest(line) + , TParsedHttpLocation(Request) + { + } +}; diff --git a/library/cpp/http/misc/parsed_request_ut.cpp b/library/cpp/http/misc/parsed_request_ut.cpp index da6d95c6ab..d578f69f24 100644 --- a/library/cpp/http/misc/parsed_request_ut.cpp +++ b/library/cpp/http/misc/parsed_request_ut.cpp @@ -1,28 +1,28 @@ -#include "parsed_request.h" - +#include "parsed_request.h" + #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(THttpParse) { Y_UNIT_TEST(TestParse) { - TParsedHttpFull h("GET /yandsearch?text=nokia HTTP/1.1"); - - UNIT_ASSERT_EQUAL(h.Method, "GET"); - UNIT_ASSERT_EQUAL(h.Request, "/yandsearch?text=nokia"); - UNIT_ASSERT_EQUAL(h.Proto, "HTTP/1.1"); - - UNIT_ASSERT_EQUAL(h.Path, "/yandsearch"); - UNIT_ASSERT_EQUAL(h.Cgi, "text=nokia"); - } - + TParsedHttpFull h("GET /yandsearch?text=nokia HTTP/1.1"); + + UNIT_ASSERT_EQUAL(h.Method, "GET"); + UNIT_ASSERT_EQUAL(h.Request, "/yandsearch?text=nokia"); + UNIT_ASSERT_EQUAL(h.Proto, "HTTP/1.1"); + + UNIT_ASSERT_EQUAL(h.Path, "/yandsearch"); + UNIT_ASSERT_EQUAL(h.Cgi, "text=nokia"); + } + Y_UNIT_TEST(TestError) { - bool wasError = false; - - try { - TParsedHttpFull("GET /yandsearch?text=nokiaHTTP/1.1"); - } catch (...) { - wasError = true; - } - - UNIT_ASSERT(wasError); - } -} + bool wasError = false; + + try { + TParsedHttpFull("GET /yandsearch?text=nokiaHTTP/1.1"); + } catch (...) { + wasError = true; + } + + UNIT_ASSERT(wasError); + } +} diff --git a/library/cpp/http/misc/ut/ya.make b/library/cpp/http/misc/ut/ya.make index f4bdd35662..49ca2b9aaf 100644 --- a/library/cpp/http/misc/ut/ya.make +++ b/library/cpp/http/misc/ut/ya.make @@ -1,11 +1,11 @@ UNITTEST_FOR(library/cpp/http/misc) - + OWNER(g:util) - -SRCS( - httpdate_ut.cpp - httpreqdata_ut.cpp - parsed_request_ut.cpp -) - -END() + +SRCS( + httpdate_ut.cpp + httpreqdata_ut.cpp + parsed_request_ut.cpp +) + +END() diff --git a/library/cpp/http/misc/ya.make b/library/cpp/http/misc/ya.make index fceb3cf79c..7a0f2a6e9f 100644 --- a/library/cpp/http/misc/ya.make +++ b/library/cpp/http/misc/ya.make @@ -1,24 +1,24 @@ -LIBRARY() - +LIBRARY() + OWNER( g:util mvel ) - + GENERATE_ENUM_SERIALIZATION(httpcodes.h) -SRCS( - httpcodes.cpp - httpdate.cpp - httpreqdata.cpp - parsed_request.cpp -) - +SRCS( + httpcodes.cpp + httpdate.cpp + httpreqdata.cpp + parsed_request.cpp +) + PEERDIR( library/cpp/cgiparam library/cpp/digest/lower_case ) -END() +END() RECURSE_FOR_TESTS(ut) diff --git a/library/cpp/http/server/conn.cpp b/library/cpp/http/server/conn.cpp index 38a76c4c30..dd0860a315 100644 --- a/library/cpp/http/server/conn.cpp +++ b/library/cpp/http/server/conn.cpp @@ -1,31 +1,31 @@ -#include "conn.h" - -#include <util/network/socket.h> +#include "conn.h" + +#include <util/network/socket.h> #include <util/stream/buffered.h> - -class THttpServerConn::TImpl { -public: + +class THttpServerConn::TImpl { +public: inline TImpl(const TSocket& s, size_t outputBufferSize) - : S_(s) - , SI_(S_) - , SO_(S_) + : S_(s) + , SI_(S_) + , SO_(S_) , BO_(&SO_, outputBufferSize) - , HI_(&SI_) - , HO_(&BO_, &HI_) - { - } - + , HI_(&SI_) + , HO_(&BO_, &HI_) + { + } + inline ~TImpl() { - } - + } + inline THttpInput* Input() noexcept { - return &HI_; - } - + return &HI_; + } + inline THttpOutput* Output() noexcept { - return &HO_; - } - + return &HO_; + } + inline void Reset() { if (S_ != INVALID_SOCKET) { // send RST packet to client @@ -34,35 +34,35 @@ public: } } -private: - TSocket S_; - TSocketInput SI_; - TSocketOutput SO_; - TBufferedOutput BO_; - THttpInput HI_; - THttpOutput HO_; -}; - -THttpServerConn::THttpServerConn(const TSocket& s) +private: + TSocket S_; + TSocketInput SI_; + TSocketOutput SO_; + TBufferedOutput BO_; + THttpInput HI_; + THttpOutput HO_; +}; + +THttpServerConn::THttpServerConn(const TSocket& s) : THttpServerConn(s, s.MaximumTransferUnit()) -{ -} - +{ +} + THttpServerConn::THttpServerConn(const TSocket& s, size_t outputBufferSize) : Impl_(new TImpl(s, outputBufferSize)) { } THttpServerConn::~THttpServerConn() { -} - +} + THttpInput* THttpServerConn::Input() noexcept { - return Impl_->Input(); -} - + return Impl_->Input(); +} + THttpOutput* THttpServerConn::Output() noexcept { - return Impl_->Output(); -} + return Impl_->Output(); +} void THttpServerConn::Reset() { return Impl_->Reset(); diff --git a/library/cpp/http/server/conn.h b/library/cpp/http/server/conn.h index 3aa5329af4..93c2358c23 100644 --- a/library/cpp/http/server/conn.h +++ b/library/cpp/http/server/conn.h @@ -1,37 +1,37 @@ -#pragma once - +#pragma once + #include <library/cpp/http/io/stream.h> -#include <util/generic/ptr.h> - -class TSocket; - -/// Потоки ввода/вывода для получения запросов и отправки ответов HTTP-сервера. -class THttpServerConn { -public: - explicit THttpServerConn(const TSocket& s); - THttpServerConn(const TSocket& s, size_t outputBufferSize); +#include <util/generic/ptr.h> + +class TSocket; + +/// Потоки ввода/вывода для получения запросов и отправки ответов HTTP-сервера. +class THttpServerConn { +public: + explicit THttpServerConn(const TSocket& s); + THttpServerConn(const TSocket& s, size_t outputBufferSize); ~THttpServerConn(); - + THttpInput* Input() noexcept; THttpOutput* Output() noexcept; - + inline const THttpInput* Input() const noexcept { - return const_cast<THttpServerConn*>(this)->Input(); - } - + return const_cast<THttpServerConn*>(this)->Input(); + } + inline const THttpOutput* Output() const noexcept { - return const_cast<THttpServerConn*>(this)->Output(); - } - - /// Проверяет, можно ли установить режим, при котором соединение с сервером - /// не завершается после окончания транзакции. + return const_cast<THttpServerConn*>(this)->Output(); + } + + /// Проверяет, можно ли установить режим, при котором соединение с сервером + /// не завершается после окончания транзакции. inline bool CanBeKeepAlive() const noexcept { - return Output()->CanBeKeepAlive(); - } - + return Output()->CanBeKeepAlive(); + } + void Reset(); -private: - class TImpl; - THolder<TImpl> Impl_; -}; +private: + class TImpl; + THolder<TImpl> Impl_; +}; diff --git a/library/cpp/http/server/http.cpp b/library/cpp/http/server/http.cpp index 128583bdd7..581fc77399 100644 --- a/library/cpp/http/server/http.cpp +++ b/library/cpp/http/server/http.cpp @@ -1,12 +1,12 @@ #include "http.h" #include "http_ex.h" - + #include <library/cpp/threading/equeue/equeue.h> #include <util/generic/buffer.h> #include <util/generic/cast.h> -#include <util/generic/intrlist.h> -#include <util/generic/yexception.h> +#include <util/generic/intrlist.h> +#include <util/generic/yexception.h> #include <util/network/address.h> #include <util/network/socket.h> #include <util/network/poller.h> @@ -18,7 +18,7 @@ #include <util/system/pipe.h> #include <util/system/thread.h> #include <util/thread/factory.h> - + #include <cerrno> #include <cstring> #include <ctime> @@ -28,39 +28,39 @@ using namespace NAddr; -namespace { - class IPollAble { - public: +namespace { + class IPollAble { + public: inline IPollAble() noexcept { - } - - virtual ~IPollAble() { - } - + } + + virtual ~IPollAble() { + } + virtual void OnPollEvent(TInstant now) = 0; - }; + }; + + struct TShouldStop { + }; - struct TShouldStop { - }; - - struct TWakeupPollAble: public IPollAble { + struct TWakeupPollAble: public IPollAble { void OnPollEvent(TInstant) override { - throw TShouldStop(); - } - }; -} - -class TClientConnection: public IPollAble, public TIntrusiveListItem<TClientConnection> { + throw TShouldStop(); + } + }; +} + +class TClientConnection: public IPollAble, public TIntrusiveListItem<TClientConnection> { public: TClientConnection(const TSocket& s, THttpServer::TImpl* serv, NAddr::IRemoteAddrRef listenerSockAddrRef); ~TClientConnection() override; - + void OnPollEvent(TInstant now) override; - + inline void Activate(TInstant now) noexcept; inline void DeActivate(); inline void Reject(); - + public: TSocket Socket_; NAddr::IRemoteAddrRef ListenerSockAddrRef_; @@ -69,28 +69,28 @@ public: TInstant LastUsed; TInstant AcceptMoment; size_t ReceivedRequests = 0; -}; - -class THttpServer::TImpl { +}; + +class THttpServer::TImpl { public: class TConnections { - public: + public: inline TConnections(TSocketPoller* poller, const THttpServerOptions& options) : Poller_(poller) , Options(options) { } - + inline ~TConnections() { } - + inline void Add(TClientConnection* c) noexcept { TGuard<TMutex> g(Mutex_); - + Conns_.PushBack(c); Poller_->WaitRead(c->Socket_, (void*)static_cast<const IPollAble*>(c)); } - + inline void Erase(TClientConnection* c, TInstant now) noexcept { TGuard<TMutex> g(Mutex_); EraseUnsafe(c); @@ -98,18 +98,18 @@ public: TryRemovingUnsafe(now - Options.ExpirationTimeout); } } - + inline void Clear() noexcept { TGuard<TMutex> g(Mutex_); - + Conns_.Clear(); } - + inline bool RemoveOld(TInstant border) noexcept { TGuard<TMutex> g(Mutex_); return TryRemovingUnsafe(border); } - + bool TryRemovingUnsafe(TInstant border) noexcept { if (Conns_.Empty()) { return false; @@ -122,7 +122,7 @@ public: delete c; return true; } - + void EraseUnsafe(TClientConnection* c) noexcept { Poller_->Unwait(c->Socket_); c->Unlink(); @@ -134,7 +134,7 @@ public: TSocketPoller* Poller_ = nullptr; const THttpServerOptions& Options; }; - + static void* ListenSocketFunction(void* param) { try { ((TImpl*)param)->ListenSocket(); @@ -144,10 +144,10 @@ public: return nullptr; } - + TAutoPtr<TClientRequest> CreateRequest(TAutoPtr<TClientConnection> c) { THolder<TClientRequest> obj(Cb_->CreateClient()); - + obj->Conn_.Reset(c.Release()); return obj; @@ -161,13 +161,13 @@ public: (new TClientConnection(s, this, listenerSockAddrRef))->Reject(); return; } - } - + } + auto connection = new TClientConnection(s, this, listenerSockAddrRef); connection->LastUsed = now; connection->DeActivate(); } - + void SaveErrorCode() { ErrorCode = WSAGetLastError(); } @@ -175,11 +175,11 @@ public: int GetErrorCode() const { return ErrorCode; } - + const char* GetError() const { return LastSystemErrorText(ErrorCode); } - + bool Start() { Poller.Reset(new TSocketPoller()); Connections.Reset(new TConnections(Poller.Get(), Options_)); @@ -202,14 +202,14 @@ public: } catch (const yexception&) { SaveErrorCode(); return false; - } - + } + // Wait until the thread has completely started and return the success indicator ListenStartEvent.Wait(); - + return ListenerRunningOK; } - + void Wait() { Cb_->OnWait(); TGuard<TMutex> g(StopMutex); @@ -227,46 +227,46 @@ public: ListenThread->Join(); ListenThread.Reset(nullptr); } - + while (ConnectionCount) { usleep(10000); Connections->Clear(); - } - + } + Connections.Destroy(); Poller.Destroy(); } - + void Shutdown() { ListenWakeupWriteFd.Write("", 1); // ignore result } - + void AddRequest(TAutoPtr<TClientRequest> req, bool fail) { struct TFailRequest: public THttpClientRequestEx { - inline TFailRequest(TAutoPtr<TClientRequest> parent) { - Conn_.Reset(parent->Conn_.Release()); - HttpConn_.Reset(parent->HttpConn_.Release()); - } - + inline TFailRequest(TAutoPtr<TClientRequest> parent) { + Conn_.Reset(parent->Conn_.Release()); + HttpConn_.Reset(parent->HttpConn_.Release()); + } + bool Reply(void*) override { if (!ProcessHeaders()) { return true; } - ProcessFailRequest(0); - return true; - } + ProcessFailRequest(0); + return true; + } }; - + if (!fail && Requests->Add(req.Get())) { Y_UNUSED(req.Release()); } else { req = new TFailRequest(req); - + if (FailRequests->Add(req.Get())) { Y_UNUSED(req.Release()); - } else { + } else { Cb_->OnFailRequest(-1); } } @@ -288,44 +288,44 @@ public: return *FailRequests; } - class TListenSocket: public IPollAble, public TIntrusiveListItem<TListenSocket> { - public: - inline TListenSocket(const TSocket& s, TImpl* parent) - : S_(s) - , Server_(parent) + class TListenSocket: public IPollAble, public TIntrusiveListItem<TListenSocket> { + public: + inline TListenSocket(const TSocket& s, TImpl* parent) + : S_(s) + , Server_(parent) , SockAddrRef_(GetSockAddr(S_)) - { - } - + { + } + ~TListenSocket() override { - } - + } + void OnPollEvent(TInstant) override { SOCKET s = ::accept(S_, nullptr, nullptr); - - if (s == INVALID_SOCKET) { - ythrow yexception() << "accept: " << LastSystemErrorText(); + + if (s == INVALID_SOCKET) { + ythrow yexception() << "accept: " << LastSystemErrorText(); } Server_->AddRequestFromSocket(s, TInstant::Now(), SockAddrRef_); - } + } SOCKET GetSocket() const noexcept { - return S_; - } - - private: - TSocket S_; + return S_; + } + + private: + TSocket S_; TImpl* Server_ = nullptr; NAddr::IRemoteAddrRef SockAddrRef_; }; - + void ListenSocket() { TThread::SetCurrentThreadName(Options_.ListenThreadName.c_str()); ErrorCode = 0; TIntrusiveListWithAutoDelete<TListenSocket, TDelete> Reqs; - + std::function<void(TSocket)> callback = [&](TSocket socket) { THolder<TListenSocket> ls(new TListenSocket(socket, this)); Poller->WaitRead(socket, static_cast<IPollAble*>(ls.Get())); @@ -335,7 +335,7 @@ public: if (!addressesBound) { SaveErrorCode(); ListenStartEvent.Signal(); - + return; } @@ -347,18 +347,18 @@ public: TVector<void*> events; events.resize(1); - + TInstant now = TInstant::Now(); - for (;;) { + for (;;) { try { const TInstant deadline = Options_.PollTimeout == TDuration::Zero() ? TInstant::Max() : now + Options_.PollTimeout; const size_t ret = Poller->WaitD(events.data(), events.size(), deadline); - + now = TInstant::Now(); for (size_t i = 0; i < ret; ++i) { ((IPollAble*)events[i])->OnPollEvent(now); } - + if (ret == 0 && Options_.ExpirationTimeout > TDuration::Zero()) { Connections->RemoveOld(now - Options_.ExpirationTimeout); } @@ -370,26 +370,26 @@ public: if (!Options_.MaxConnections && Options_.ExpirationTimeout == TDuration::Zero()) { if (ret >= events.size()) { events.resize(ret * 2); - } - } - } catch (const TShouldStop&) { - break; + } + } + } catch (const TShouldStop&) { + break; } catch (...) { Cb_->OnException(); - } - } + } + } while (!Reqs.Empty()) { THolder<TListenSocket> ls(Reqs.PopFront()); - + Poller->Unwait(ls->GetSocket()); - } - + } + Requests->Stop(); FailRequests->Stop(); Cb_->OnListenStop(); } - + void RestartRequestThreads(ui32 nTh, ui32 maxQS) { Requests->Stop(); Options_.nThreads = nTh; @@ -408,24 +408,24 @@ public: TImpl(THttpServer* parent, ICallBack* cb, const TOptions& options, IThreadFactory* factory) : TImpl( - parent, - cb, + parent, + cb, MakeThreadPool<TSimpleThreadPool>(factory, options.UseElasticQueues, cb, options.RequestsThreadName), MakeThreadPool<TThreadPool>(factory, options.UseElasticQueues, nullptr, options.FailRequestsThreadName), - options) { + options) { } ~TImpl() { try { Stop(); } catch (...) { - } + } } - + inline const TOptions& Options() const noexcept { return Options_; } - + inline void DecreaseConnections() noexcept { AtomicDecrement(ConnectionCount); } @@ -439,7 +439,7 @@ public: } inline bool MaxRequestsReached() const { - return Options_.MaxConnections && ((size_t)GetClientCount() >= Options_.MaxConnections); + return Options_.MaxConnections && ((size_t)GetClientCount() >= Options_.MaxConnections); } THolder<TThread> ListenThread; @@ -480,57 +480,57 @@ private: return pool; } -}; +}; THttpServer::THttpServer(ICallBack* cb, const TOptions& options, IThreadFactory* pool) : Impl_(new TImpl(this, cb, options, pool)) -{ -} - +{ +} + THttpServer::THttpServer(ICallBack* cb, TMtpQueueRef mainWorkers, TMtpQueueRef failWorkers, const TOptions& options) : Impl_(new TImpl(this, cb, mainWorkers, failWorkers, options)) { } -THttpServer::~THttpServer() { -} - +THttpServer::~THttpServer() { +} + i64 THttpServer::GetClientCount() const { return Impl_->GetClientCount(); } -bool THttpServer::Start() { - return Impl_->Start(); -} - -void THttpServer::Stop() { - Impl_->Stop(); -} - -void THttpServer::Shutdown() { - Impl_->Shutdown(); -} - -void THttpServer::Wait() { - Impl_->Wait(); -} - -int THttpServer::GetErrorCode() { - return Impl_->GetErrorCode(); -} - -const char* THttpServer::GetError() { - return Impl_->GetError(); -} - -void THttpServer::RestartRequestThreads(ui32 n, ui32 queue) { - Impl_->RestartRequestThreads(n, queue); -} - +bool THttpServer::Start() { + return Impl_->Start(); +} + +void THttpServer::Stop() { + Impl_->Stop(); +} + +void THttpServer::Shutdown() { + Impl_->Shutdown(); +} + +void THttpServer::Wait() { + Impl_->Wait(); +} + +int THttpServer::GetErrorCode() { + return Impl_->GetErrorCode(); +} + +const char* THttpServer::GetError() { + return Impl_->GetError(); +} + +void THttpServer::RestartRequestThreads(ui32 n, ui32 queue) { + Impl_->RestartRequestThreads(n, queue); +} + const THttpServer::TOptions& THttpServer::Options() const noexcept { - return Impl_->Options(); -} - + return Impl_->Options(); +} + size_t THttpServer::GetRequestQueueSize() const { return Impl_->GetRequestQueueSize(); } @@ -552,45 +552,45 @@ bool THttpServer::MaxRequestsReached() const { } TClientConnection::TClientConnection(const TSocket& s, THttpServer::TImpl* serv, NAddr::IRemoteAddrRef listenerSockAddrRef) - : Socket_(s) + : Socket_(s) , ListenerSockAddrRef_(listenerSockAddrRef) - , HttpServ_(serv) -{ - SetNoDelay(Socket_, true); - + , HttpServ_(serv) +{ + SetNoDelay(Socket_, true); + const TDuration& clientTimeout = HttpServ_->Options().ClientTimeout; if (clientTimeout != TDuration::Zero()) { SetSocketTimeout(Socket_, (long)clientTimeout.Seconds(), clientTimeout.MilliSecondsOfSecond()); } - HttpServ_->IncreaseConnections(); -} - + HttpServ_->IncreaseConnections(); +} + TClientConnection::~TClientConnection() { - HttpServ_->DecreaseConnections(); -} - + HttpServ_->DecreaseConnections(); +} + void TClientConnection::OnPollEvent(TInstant now) { THolder<TClientConnection> this_(this); Activate(now); - - { - char tmp[1]; - - if (::recv(Socket_, tmp, 1, MSG_PEEK) < 1) { - /* - * We can received a FIN so our socket was moved to - * TCP_CLOSE_WAIT state. Check it before adding work - * for this socket. - */ - - return; - } - } - + + { + char tmp[1]; + + if (::recv(Socket_, tmp, 1, MSG_PEEK) < 1) { + /* + * We can received a FIN so our socket was moved to + * TCP_CLOSE_WAIT state. Check it before adding work + * for this socket. + */ + + return; + } + } + THolder<TClientRequest> obj(HttpServ_->CreateRequest(this_)); AcceptMoment = now; - + HttpServ_->AddRequest(obj, Reject_); } @@ -600,35 +600,35 @@ void TClientConnection::Activate(TInstant now) noexcept { ++ReceivedRequests; } -void TClientConnection::DeActivate() { +void TClientConnection::DeActivate() { HttpServ_->Connections->Add(this); -} - +} + void TClientConnection::Reject() { Reject_ = true; HttpServ_->Connections->Add(this); } -TClientRequest::TClientRequest() { -} - -TClientRequest::~TClientRequest() { -} - -bool TClientRequest::Reply(void* /*ThreadSpecificResource*/) { +TClientRequest::TClientRequest() { +} + +TClientRequest::~TClientRequest() { +} + +bool TClientRequest::Reply(void* /*ThreadSpecificResource*/) { if (strnicmp(RequestString.data(), "GET ", 4)) { - Output() << "HTTP/1.0 501 Not Implemented\r\n\r\n"; - } else { - Output() << "HTTP/1.0 200 OK\r\n" - "Content-Type: text/html\r\n" - "\r\n" - "Hello World!\r\n"; - } - - return true; -} - + Output() << "HTTP/1.0 501 Not Implemented\r\n\r\n"; + } else { + Output() << "HTTP/1.0 200 OK\r\n" + "Content-Type: text/html\r\n" + "\r\n" + "Hello World!\r\n"; + } + + return true; +} + bool TClientRequest::IsLocal() const { return HasLocalAddress(Socket()); } @@ -639,29 +639,29 @@ bool TClientRequest::CheckLoopback() { try { isLocal = IsLocal(); } catch (const yexception& e) { - Output() << "HTTP/1.0 500 Oops\r\n\r\n" - << e.what() << "\r\n"; - return false; + Output() << "HTTP/1.0 500 Oops\r\n\r\n" + << e.what() << "\r\n"; + return false; } if (!isLocal) { - Output() << "HTTP/1.0 403 Permission denied\r\n" - "Content-Type: text/html; charset=windows-1251\r\n" - "Connection: close\r\n" - "\r\n" - "<html><head><title>Permission denied</title></head>" - "<body><h1>Permission denied</h1>" - "<p>This request must be sent from the localhost.</p>" - "</body></html>\r\n"; - - return false; - } - - return true; -} - + Output() << "HTTP/1.0 403 Permission denied\r\n" + "Content-Type: text/html; charset=windows-1251\r\n" + "Connection: close\r\n" + "\r\n" + "<html><head><title>Permission denied</title></head>" + "<body><h1>Permission denied</h1>" + "<p>This request must be sent from the localhost.</p>" + "</body></html>\r\n"; + + return false; + } + + return true; +} + void TClientRequest::ReleaseConnection() { - if (Conn_ && HttpConn_ && HttpServ()->Options().KeepAliveEnabled && HttpConn_->CanBeKeepAlive() && (!HttpServ()->Options().RejectExcessConnections || !HttpServ()->MaxRequestsReached())) { + if (Conn_ && HttpConn_ && HttpServ()->Options().KeepAliveEnabled && HttpConn_->CanBeKeepAlive() && (!HttpServ()->Options().RejectExcessConnections || !HttpServ()->MaxRequestsReached())) { Output().Finish(); Conn_->DeActivate(); Y_UNUSED(Conn_.Release()); @@ -676,101 +676,101 @@ void TClientRequest::ResetConnection() { } } -void TClientRequest::Process(void* ThreadSpecificResource) { +void TClientRequest::Process(void* ThreadSpecificResource) { THolder<TClientRequest> this_(this); - + auto* serverImpl = Conn_->HttpServ_; - try { - if (!HttpConn_) { + try { + if (!HttpConn_) { const size_t outputBufferSize = HttpServ()->Options().OutputBufferSize; if (outputBufferSize) { HttpConn_.Reset(new THttpServerConn(Socket(), outputBufferSize)); } else { HttpConn_.Reset(new THttpServerConn(Socket())); } - + auto maxRequestsPerConnection = HttpServ()->Options().MaxRequestsPerConnection; HttpConn_->Output()->EnableKeepAlive(HttpServ()->Options().KeepAliveEnabled && (!maxRequestsPerConnection || Conn_->ReceivedRequests < maxRequestsPerConnection)); - HttpConn_->Output()->EnableCompression(HttpServ()->Options().CompressionEnabled); - } - + HttpConn_->Output()->EnableCompression(HttpServ()->Options().CompressionEnabled); + } + if (ParsedHeaders.empty()) { RequestString = Input().FirstLine(); - - const THttpHeaders& h = Input().Headers(); + + const THttpHeaders& h = Input().Headers(); ParsedHeaders.reserve(h.Count()); - for (THttpHeaders::TConstIterator it = h.Begin(); it != h.End(); ++it) { + for (THttpHeaders::TConstIterator it = h.Begin(); it != h.End(); ++it) { ParsedHeaders.emplace_back(it->Name(), it->Value()); - } - } - - if (Reply(ThreadSpecificResource)) { + } + } + + if (Reply(ThreadSpecificResource)) { ReleaseConnection(); - - /* - * *this will be destroyed... - */ - - return; - } - } catch (...) { + + /* + * *this will be destroyed... + */ + + return; + } + } catch (...) { serverImpl->Cb_->OnException(); - - throw; - } - + + throw; + } + Y_UNUSED(this_.Release()); -} - -void TClientRequest::ProcessFailRequest(int failstate) { +} + +void TClientRequest::ProcessFailRequest(int failstate) { Output() << "HTTP/1.1 503 Service Unavailable\r\n" "Content-Type: text/plain\r\n" "Content-Length: 21\r\n" "\r\n" "Service Unavailable\r\n"; - + TString url; - + if (!strnicmp(RequestString.data(), "GET ", 4)) { - // Trying to extract url... + // Trying to extract url... const char* str = RequestString.data(); - - // Skipping spaces before url... - size_t start = 3; + + // Skipping spaces before url... + size_t start = 3; while (str[start] == ' ') { - ++start; - } - - if (str[start]) { - // Traversing url... - size_t idx = start; - - while (str[idx] != ' ' && str[idx]) { - ++idx; - } - + ++start; + } + + if (str[start]) { + // Traversing url... + size_t idx = start; + + while (str[idx] != ' ' && str[idx]) { + ++idx; + } + url = RequestString.substr(start, idx - start); - } - } - - const THttpServer::ICallBack::TFailLogData d = { - failstate, - url, - }; - - // Handling failure... - Conn_->HttpServ_->Cb_->OnFailRequestEx(d); - Output().Flush(); -} - + } + } + + const THttpServer::ICallBack::TFailLogData d = { + failstate, + url, + }; + + // Handling failure... + Conn_->HttpServ_->Cb_->OnFailRequestEx(d); + Output().Flush(); +} + THttpServer* TClientRequest::HttpServ() const noexcept { - return Conn_->HttpServ_->Parent_; -} - + return Conn_->HttpServ_->Parent_; +} + const TSocket& TClientRequest::Socket() const noexcept { - return Conn_->Socket_; -} + return Conn_->Socket_; +} NAddr::IRemoteAddrRef TClientRequest::GetListenerSockAddrRef() const noexcept { return Conn_->ListenerSockAddrRef_; @@ -791,8 +791,8 @@ TRequestReplier::~TRequestReplier() { bool TRequestReplier::Reply(void* threadSpecificResource) { const TReplyParams params = { - threadSpecificResource, Input(), Output()}; - + threadSpecificResource, Input(), Output()}; + return DoReply(params); } @@ -833,7 +833,7 @@ bool TryToBindAddresses(const THttpServerOptions& options, const std::function<v return false; } - if (callbackOnBoundAddress != nullptr) { + if (callbackOnBoundAddress != nullptr) { (*callbackOnBoundAddress)(socket); } } diff --git a/library/cpp/http/server/http.h b/library/cpp/http/server/http.h index b292d38f27..1b72215706 100644 --- a/library/cpp/http/server/http.h +++ b/library/cpp/http/server/http.h @@ -1,92 +1,92 @@ #pragma once -#include "conn.h" -#include "options.h" - +#include "conn.h" +#include "options.h" + #include <util/thread/pool.h> #include <library/cpp/http/io/stream.h> -#include <util/memory/blob.h> +#include <util/memory/blob.h> #include <util/generic/ptr.h> #include <util/generic/vector.h> #include <util/system/atomic.h> - + class IThreadFactory; -class TClientRequest; -class TClientConnection; +class TClientRequest; +class TClientConnection; -class THttpServer { +class THttpServer { friend class TClientRequest; friend class TClientConnection; - + public: class ICallBack { - public: + public: struct TFailLogData { int failstate; TString url; }; - + virtual ~ICallBack() { } - + virtual void OnFailRequest(int /*failstate*/) { } - + virtual void OnFailRequestEx(const TFailLogData& d) { OnFailRequest(d.failstate); } - + virtual void OnException() { } - + virtual void OnMaxConn() { } - + virtual TClientRequest* CreateClient() = 0; - + virtual void OnListenStart() { } - + virtual void OnListenStop() { } - + virtual void OnWait() { } - - virtual void* CreateThreadSpecificResource() { + + virtual void* CreateThreadSpecificResource() { return nullptr; } - + virtual void DestroyThreadSpecificResource(void*) { } }; - + typedef THttpServerOptions TOptions; typedef TSimpleSharedPtr<IThreadPool> TMtpQueueRef; - + THttpServer(ICallBack* cb, const TOptions& options = TOptions(), IThreadFactory* pool = nullptr); THttpServer(ICallBack* cb, TMtpQueueRef mainWorkers, TMtpQueueRef failWorkers, const TOptions& options = TOptions()); virtual ~THttpServer(); - + bool Start(); - + // shutdown a.s.a.p. void Stop(); - + // graceful shutdown with serving all already open connections void Shutdown(); - + void Wait(); int GetErrorCode(); const char* GetError(); void RestartRequestThreads(ui32 nTh, ui32 maxQS); const TOptions& Options() const noexcept; i64 GetClientCount() const; - + class TImpl; size_t GetRequestQueueSize() const; size_t GetFailQueueSize() const; - + const IThreadPool& GetRequestQueue() const; const IThreadPool& GetFailQueue() const; @@ -102,32 +102,32 @@ private: /** * @deprecated Use TRequestReplier instead */ -class TClientRequest: public IObjectInQueue { +class TClientRequest: public IObjectInQueue { friend class THttpServer::TImpl; - + public: TClientRequest(); ~TClientRequest() override; - + inline THttpInput& Input() noexcept { return *HttpConn_->Input(); } - + inline THttpOutput& Output() noexcept { return *HttpConn_->Output(); } - + THttpServer* HttpServ() const noexcept; const TSocket& Socket() const noexcept; NAddr::IRemoteAddrRef GetListenerSockAddrRef() const noexcept; TInstant AcceptMoment() const noexcept; - + bool IsLocal() const; bool CheckLoopback(); void ProcessFailRequest(int failstate); - + void ReleaseConnection(); - + void ResetConnection(); private: @@ -138,17 +138,17 @@ private: */ virtual bool Reply(void* ThreadSpecificResource); void Process(void* ThreadSpecificResource) override; - + public: TVector<std::pair<TString, TString>> ParsedHeaders; TString RequestString; - + private: THolder<TClientConnection> Conn_; THolder<THttpServerConn> HttpConn_; -}; - -class TRequestReplier: public TClientRequest { +}; + +class TRequestReplier: public TClientRequest { public: TRequestReplier(); ~TRequestReplier() override; diff --git a/library/cpp/http/server/http_ex.cpp b/library/cpp/http/server/http_ex.cpp index e07db22bfc..2fa2ccfead 100644 --- a/library/cpp/http/server/http_ex.cpp +++ b/library/cpp/http/server/http_ex.cpp @@ -1,32 +1,32 @@ -#include "http_ex.h" - -#include <util/generic/buffer.h> -#include <util/generic/cast.h> +#include "http_ex.h" + +#include <util/generic/buffer.h> +#include <util/generic/cast.h> #include <util/stream/null.h> - + bool THttpClientRequestExtension::Parse(char* req, TBaseServerRequestData& rd) { rd.SetSocket(Socket()); - + if (!rd.Parse(req)) { - Output() << "HTTP/1.1 403 Forbidden\r\n" - "Content-Type: text/plain\r\n" - "Content-Length: 39\r\n" - "\r\n" - "The server cannot be used as a proxy.\r\n"; - - return false; - } - - return true; -} - + Output() << "HTTP/1.1 403 Forbidden\r\n" + "Content-Type: text/plain\r\n" + "Content-Length: 39\r\n" + "\r\n" + "The server cannot be used as a proxy.\r\n"; + + return false; + } + + return true; +} + bool THttpClientRequestExtension::ProcessHeaders(TBaseServerRequestData& rd, TBlob& postData) { - for (const auto& header : ParsedHeaders) { + for (const auto& header : ParsedHeaders) { rd.AddHeader(header.first, header.second); - } - + } + char* s = RequestString.begin(); - + enum EMethod { NotImplemented, Get, @@ -39,7 +39,7 @@ bool THttpClientRequestExtension::ProcessHeaders(TBaseServerRequestData& rd, TBl enum EMethod foundMethod; char* urlStart; - if (strnicmp(s, "GET ", 4) == 0) { + if (strnicmp(s, "GET ", 4) == 0) { foundMethod = Get; urlStart = s + 4; } else if (strnicmp(s, "POST ", 5) == 0) { @@ -59,49 +59,49 @@ bool THttpClientRequestExtension::ProcessHeaders(TBaseServerRequestData& rd, TBl } switch (foundMethod) { - case Get: - case Delete: - if (!Parse(urlStart, rd)) { - return false; - } - break; - - case Post: - case Put: + case Get: + case Delete: + if (!Parse(urlStart, rd)) { + return false; + } + break; + + case Post: + case Put: case Patch: - try { - ui64 contentLength = 0; - if (Input().HasExpect100Continue()) { - Output().SendContinue(); - } - - if (!Input().ContentEncoded() && Input().GetContentLength(contentLength)) { + try { + ui64 contentLength = 0; + if (Input().HasExpect100Continue()) { + Output().SendContinue(); + } + + if (!Input().ContentEncoded() && Input().GetContentLength(contentLength)) { if (contentLength > HttpServ()->Options().MaxInputContentLength) { Output() << "HTTP/1.1 413 Payload Too Large\r\nContent-Length:0\r\n\r\n"; Output().Finish(); return false; } - TBuffer buf(SafeIntegerCast<size_t>(contentLength)); - buf.Resize(Input().Load(buf.Data(), (size_t)contentLength)); - postData = TBlob::FromBuffer(buf); - } else { - postData = TBlob::FromStream(Input()); - } - } catch (...) { - Output() << "HTTP/1.1 400 Bad request\r\n\r\n"; - return false; + TBuffer buf(SafeIntegerCast<size_t>(contentLength)); + buf.Resize(Input().Load(buf.Data(), (size_t)contentLength)); + postData = TBlob::FromBuffer(buf); + } else { + postData = TBlob::FromStream(Input()); + } + } catch (...) { + Output() << "HTTP/1.1 400 Bad request\r\n\r\n"; + return false; } - if (!Parse(urlStart, rd)) { - return false; - } - break; - - case NotImplemented: - Output() << "HTTP/1.1 501 Not Implemented\r\n\r\n"; - return false; - } - - return true; -} + if (!Parse(urlStart, rd)) { + return false; + } + break; + + case NotImplemented: + Output() << "HTTP/1.1 501 Not Implemented\r\n\r\n"; + return false; + } + + return true; +} diff --git a/library/cpp/http/server/http_ex.h b/library/cpp/http/server/http_ex.h index 1ef43ea4fd..39cfbbe66f 100644 --- a/library/cpp/http/server/http_ex.h +++ b/library/cpp/http/server/http_ex.h @@ -1,28 +1,28 @@ -#pragma once - -#include "http.h" - +#pragma once + +#include "http.h" + #include <library/cpp/http/misc/httpreqdata.h> - -class THttpClientRequestExtension: public TClientRequest { + +class THttpClientRequestExtension: public TClientRequest { public: bool Parse(char* req, TBaseServerRequestData& rd); bool ProcessHeaders(TBaseServerRequestData& rd, TBlob& postData); }; - + template <class TRequestData> -class THttpClientRequestExtImpl: public THttpClientRequestExtension { -protected: +class THttpClientRequestExtImpl: public THttpClientRequestExtension { +protected: bool Parse(char* req) { return THttpClientRequestExtension::Parse(req, RD); } bool ProcessHeaders() { return THttpClientRequestExtension::ProcessHeaders(RD, Buf); } - + protected: TRequestData RD; - TBlob Buf; -}; + TBlob Buf; +}; using THttpClientRequestEx = THttpClientRequestExtImpl<TServerRequestData>; diff --git a/library/cpp/http/server/http_ut.cpp b/library/cpp/http/server/http_ut.cpp index cc62bb988e..ab24207eac 100644 --- a/library/cpp/http/server/http_ut.cpp +++ b/library/cpp/http/server/http_ut.cpp @@ -1,58 +1,58 @@ -#include "http.h" -#include "http_ex.h" - +#include "http.h" +#include "http_ex.h" + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/testing/unittest/tests_data.h> - + #include <util/generic/cast.h> #include <util/stream/output.h> #include <util/stream/zlib.h> #include <util/system/datetime.h> #include <util/system/sem.h> - + Y_UNIT_TEST_SUITE(THttpServerTest) { - class TEchoServer: public THttpServer::ICallBack { + class TEchoServer: public THttpServer::ICallBack { class TRequest: public THttpClientRequestEx { - public: - inline TRequest(TEchoServer* parent) - : Parent_(parent) - { - } - + public: + inline TRequest(TEchoServer* parent) + : Parent_(parent) + { + } + bool Reply(void* /*tsr*/) override { - if (!ProcessHeaders()) { + if (!ProcessHeaders()) { return true; - } - - Output() << "HTTP/1.1 200 Ok\r\n\r\n"; + } + + Output() << "HTTP/1.1 200 Ok\r\n\r\n"; if (Buf.Size()) { Output().Write(Buf.AsCharPtr(), Buf.Size()); - } else { - Output() << Parent_->Res_; - } - Output().Finish(); - - return true; - } - - private: + } else { + Output() << Parent_->Res_; + } + Output().Finish(); + + return true; + } + + private: TEchoServer* Parent_ = nullptr; - }; - - public: + }; + + public: inline TEchoServer(const TString& res) - : Res_(res) - { - } - + : Res_(res) + { + } + TClientRequest* CreateClient() override { - return new TRequest(this); - } - - private: + return new TRequest(this); + } + + private: TString Res_; - }; - + }; + class TSleepingServer: public THttpServer::ICallBack { class TReplier: public TRequestReplier { public: @@ -269,7 +269,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { TStringStream ss; ss << (KeepAliveConnection ? "keep-alive " : "") << Type; if (ContentEncoding.size()) { - ss << " with encoding=" << ContentEncoding; + ss << " with encoding=" << ContentEncoding; } return ss.Str(); } else { @@ -292,7 +292,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { class TFailingMtpQueue: public TSimpleThreadPool { private: bool FailOnAdd_ = false; - + public: void SetFailOnAdd(bool fail = true) { FailOnAdd_ = fail; @@ -313,47 +313,47 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { TString TestData(size_t size = 5 * 4096) { TString res; - + for (size_t i = 0; i < size; ++i) { - res += (char)i; - } + res += (char)i; + } return res; } - + Y_UNIT_TEST(TestEchoServer) { TString res = TestData(); TPortManager pm; const ui16 port = pm.GetPort(); const bool trueFalse[] = {true, false}; - - TEchoServer serverImpl(res); - THttpServer server(&serverImpl, THttpServer::TOptions(port).EnableKeepAlive(true).EnableCompression(true)); - + + TEchoServer serverImpl(res); + THttpServer server(&serverImpl, THttpServer::TOptions(port).EnableKeepAlive(true).EnableCompression(true)); + for (int i = 0; i < 2; ++i) { UNIT_ASSERT(server.Start()); - + TTestRequest r(port); r.Content = res; - - for (bool keepAlive : trueFalse) { + + for (bool keepAlive : trueFalse) { r.KeepAliveConnection = keepAlive; - + // THttpOutput use chunked stream, else use Content-Length - for (bool useHttpOutput : trueFalse) { + for (bool useHttpOutput : trueFalse) { r.UseHttpOutput = useHttpOutput; - - for (bool enableResponseEncoding : trueFalse) { + + for (bool enableResponseEncoding : trueFalse) { r.EnableResponseEncoding = enableResponseEncoding; - + const TString reqTypes[] = {"GET", "POST"}; for (const TString& reqType : reqTypes) { r.Type = reqType; - + const TString encoders[] = {"", "deflate"}; for (const TString& encoder : encoders) { r.ContentEncoding = encoder; - - for (bool expect100Continue : trueFalse) { + + for (bool expect100Continue : trueFalse) { r.Expect100Continue = expect100Continue; TString resp = r.Execute(); UNIT_ASSERT_C(resp == res, "diff echo response for request:\n" + r.GetDescription()); @@ -365,12 +365,12 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { } server.Stop(); - } - } + } + } Y_UNIT_TEST(TestReusePortEnabled) { if (!IsReusePortAvailable()) { - return; // skip test + return; // skip test } TString res = TestData(); TPortManager pm; @@ -383,7 +383,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { } for (ui32 testRun = 0; testRun < 3; testRun++) { - for (auto& server : servers) { + for (auto& server : servers) { // start servers one at a time and check at least one of them is replying UNIT_ASSERT(server->Start()); @@ -391,7 +391,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { UNIT_ASSERT_C(r.Execute() == res, "diff echo response for request:\n" + r.GetDescription()); } - for (auto& server : servers) { + for (auto& server : servers) { // ping servers and stop them one at a time // at the last iteration only one server is still working and then gets stopped as well @@ -471,7 +471,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { return true; } }; - + public: TClientRequest* CreateClient() override { return new TRequest(); @@ -684,7 +684,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { server.Stop(); }; -#if 0 +#if 0 Y_UNIT_TEST(TestSocketsLeak) { const bool trueFalse[] = {true, false}; TPortManager portManager; @@ -709,7 +709,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { server.BusyThread(); for (size_t i = 0; i < 3; ++i) { - auto func = [&server, port, keepAlive]() { + auto func = [&server, port, keepAlive]() { server.BusyThread(); THolder<TTestRequest> r = MakeHolder<TTestRequest>(port); r->KeepAliveConnection = keepAlive; @@ -722,7 +722,7 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { Sleep(TDuration::MilliSeconds(100)); server.BusyThread(); // we wait while connections are established by the // system and accepted by the server - server.Free(3); // we release all connections processing + server.Free(3); // we release all connections processing for (auto&& thread : threads) { thread->Join(); @@ -735,5 +735,5 @@ Y_UNIT_TEST_SUITE(THttpServerTest) { UNIT_ASSERT_EQUAL_C(server.RepliesCount(), 1, "only one request should have been processed, got " + ToString(server.RepliesCount())); } } -#endif -} +#endif +} diff --git a/library/cpp/http/server/options.cpp b/library/cpp/http/server/options.cpp index 05c954384a..391ac68a11 100644 --- a/library/cpp/http/server/options.cpp +++ b/library/cpp/http/server/options.cpp @@ -1,12 +1,12 @@ -#include "options.h" - +#include "options.h" + #include <util/string/cast.h> -#include <util/digest/numeric.h> -#include <util/network/ip.h> -#include <util/network/socket.h> -#include <util/generic/hash_set.h> -#include <util/generic/yexception.h> - +#include <util/digest/numeric.h> +#include <util/network/ip.h> +#include <util/network/socket.h> +#include <util/generic/hash_set.h> +#include <util/generic/yexception.h> + using TAddr = THttpServerOptions::TAddr; static inline TString AddrToString(const TAddr& addr) { @@ -16,28 +16,28 @@ static inline TString AddrToString(const TAddr& addr) { static inline TNetworkAddress ToNetworkAddr(const TString& address, ui16 port) { if (address.empty() || address == TStringBuf("*")) { return TNetworkAddress(port); - } - + } + return TNetworkAddress(address, port); -} - -void THttpServerOptions::BindAddresses(TBindAddresses& ret) const { +} + +void THttpServerOptions::BindAddresses(TBindAddresses& ret) const { THashSet<TString> check; - + for (auto addr : BindSockaddr) { if (!addr.Port) { addr.Port = Port; - } - + } + const TString straddr = AddrToString(addr); if (check.find(straddr) == check.end()) { check.insert(straddr); ret.push_back(ToNetworkAddr(addr.Addr, addr.Port)); - } - } - - if (ret.empty()) { + } + } + + if (ret.empty()) { ret.push_back(Host ? TNetworkAddress(Host, Port) : TNetworkAddress(Port)); - } -} + } +} diff --git a/library/cpp/http/server/options.h b/library/cpp/http/server/options.h index 38eda0e5e7..28ce8be0e1 100644 --- a/library/cpp/http/server/options.h +++ b/library/cpp/http/server/options.h @@ -1,49 +1,49 @@ #pragma once - -#include <util/network/ip.h> -#include <util/network/init.h> + +#include <util/network/ip.h> +#include <util/network/init.h> #include <util/network/address.h> #include <util/generic/size_literals.h> #include <util/generic/string.h> -#include <util/generic/vector.h> +#include <util/generic/vector.h> #include <util/datetime/base.h> - -class THttpServerOptions { -public: + +class THttpServerOptions { +public: inline THttpServerOptions(ui16 port = 17000) noexcept : Port(port) - { - } - + { + } + using TBindAddresses = TVector<TNetworkAddress>; - void BindAddresses(TBindAddresses& ret) const; - + void BindAddresses(TBindAddresses& ret) const; + inline THttpServerOptions& AddBindAddress(const TString& address, ui16 port) { const TAddr addr = { - address, - port, - }; - + address, + port, + }; + BindSockaddr.push_back(addr); return *this; } inline THttpServerOptions& AddBindAddress(const TString& address) { return AddBindAddress(address, 0); - } - + } + inline THttpServerOptions& EnableKeepAlive(bool enable) noexcept { - KeepAliveEnabled = enable; - - return *this; - } - + KeepAliveEnabled = enable; + + return *this; + } + inline THttpServerOptions& EnableCompression(bool enable) noexcept { - CompressionEnabled = enable; - - return *this; - } - + CompressionEnabled = enable; + + return *this; + } + inline THttpServerOptions& EnableRejectExcessConnections(bool enable) noexcept { RejectExcessConnections = enable; @@ -63,11 +63,11 @@ public: } inline THttpServerOptions& SetThreads(ui32 threads) noexcept { - nThreads = threads; - - return *this; - } - + nThreads = threads; + + return *this; + } + /// Default interface name to bind the server. Used when none of BindAddress are provided. inline THttpServerOptions& SetHost(const TString& host) noexcept { Host = host; @@ -77,35 +77,35 @@ public: /// Default port to bind the server. Used when none of BindAddress are provided. inline THttpServerOptions& SetPort(ui16 port) noexcept { - Port = port; - - return *this; - } - + Port = port; + + return *this; + } + inline THttpServerOptions& SetMaxConnections(ui32 mc = 0) noexcept { - MaxConnections = mc; - - return *this; - } - + MaxConnections = mc; + + return *this; + } + inline THttpServerOptions& SetMaxQueueSize(ui32 mqs = 0) noexcept { - MaxQueueSize = mqs; - - return *this; - } + MaxQueueSize = mqs; + + return *this; + } inline THttpServerOptions& SetClientTimeout(const TDuration& timeout) noexcept { - ClientTimeout = timeout; - - return *this; - } + ClientTimeout = timeout; + return *this; + } + inline THttpServerOptions& SetListenBacklog(int val) noexcept { - ListenBacklog = val; - - return *this; - } - + ListenBacklog = val; + + return *this; + } + inline THttpServerOptions& SetOutputBufferSize(size_t val) noexcept { OutputBufferSize = val; @@ -149,19 +149,19 @@ public: bool KeepAliveEnabled = true; bool CompressionEnabled = false; bool RejectExcessConnections = false; - bool ReusePort = false; // set SO_REUSEPORT socket option + bool ReusePort = false; // set SO_REUSEPORT socket option bool ReuseAddress = true; // set SO_REUSEADDR socket option - TAddrs BindSockaddr; - ui16 Port = 17000; // The port on which to run the web server - TString Host; // DNS entry - const char* ServerName = "YWS/1.0"; // The Web server name to return in HTTP headers - ui32 nThreads = 0; // Thread count for requests processing - ui32 MaxQueueSize = 0; // Max allowed request count in queue + TAddrs BindSockaddr; + ui16 Port = 17000; // The port on which to run the web server + TString Host; // DNS entry + const char* ServerName = "YWS/1.0"; // The Web server name to return in HTTP headers + ui32 nThreads = 0; // Thread count for requests processing + ui32 MaxQueueSize = 0; // Max allowed request count in queue ui32 nFThreads = 1; ui32 MaxFQueueSize = 0; ui32 MaxConnections = 100; int ListenBacklog = SOMAXCONN; - TDuration ClientTimeout; + TDuration ClientTimeout; size_t OutputBufferSize = 0; ui64 MaxInputContentLength = sizeof(size_t) <= 4 ? 2_GB : 64_GB; size_t MaxRequestsPerConnection = 0; // If keep-alive is enabled, request limit before connection is closed @@ -173,4 +173,4 @@ public: TString ListenThreadName = "HttpListen"; TString RequestsThreadName = "HttpServer"; TString FailRequestsThreadName = "HttpServer"; -}; +}; diff --git a/library/cpp/http/server/response.cpp b/library/cpp/http/server/response.cpp index 52d64c91ce..3c7a3e6a93 100644 --- a/library/cpp/http/server/response.cpp +++ b/library/cpp/http/server/response.cpp @@ -1,7 +1,7 @@ #include "response.h" - + #include <util/stream/output.h> -#include <util/stream/mem.h> +#include <util/stream/mem.h> #include <util/string/cast.h> THttpResponse& THttpResponse::AddMultipleHeaders(const THttpHeaders& headers) { @@ -11,16 +11,16 @@ THttpResponse& THttpResponse::AddMultipleHeaders(const THttpHeaders& headers) { return *this; } -THttpResponse& THttpResponse::SetContentType(const TStringBuf& contentType) { +THttpResponse& THttpResponse::SetContentType(const TStringBuf& contentType) { Headers.AddOrReplaceHeader(THttpInputHeader("Content-Type", ToString(contentType))); - - return *this; + + return *this; } void THttpResponse::OutTo(IOutputStream& os) const { TVector<IOutputStream::TPart> parts; const size_t FIRST_LINE_PARTS = 3; - const size_t HEADERS_PARTS = Headers.Count() * 4; + const size_t HEADERS_PARTS = Headers.Count() * 4; const size_t CONTENT_PARTS = 5; parts.reserve(FIRST_LINE_PARTS + HEADERS_PARTS + CONTENT_PARTS); @@ -30,20 +30,20 @@ void THttpResponse::OutTo(IOutputStream& os) const { parts.push_back(IOutputStream::TPart::CrLf()); // headers - for (THttpHeaders::TConstIterator i = Headers.Begin(); i != Headers.End(); ++i) { + for (THttpHeaders::TConstIterator i = Headers.Begin(); i != Headers.End(); ++i) { parts.push_back(IOutputStream::TPart(i->Name())); parts.push_back(IOutputStream::TPart(TStringBuf(": "))); parts.push_back(IOutputStream::TPart(i->Value())); parts.push_back(IOutputStream::TPart::CrLf()); } - char buf[50]; - + char buf[50]; + if (!Content.empty()) { - TMemoryOutput mo(buf, sizeof(buf)); + TMemoryOutput mo(buf, sizeof(buf)); mo << Content.size(); - + parts.push_back(IOutputStream::TPart(TStringBuf("Content-Length: "))); parts.push_back(IOutputStream::TPart(buf, mo.Buf() - buf)); parts.push_back(IOutputStream::TPart::CrLf()); @@ -51,15 +51,15 @@ void THttpResponse::OutTo(IOutputStream& os) const { // content parts.push_back(IOutputStream::TPart::CrLf()); - + if (!Content.empty()) { parts.push_back(IOutputStream::TPart(Content)); } os.Write(parts.data(), parts.size()); -} - -template <> +} + +template <> void Out<THttpResponse>(IOutputStream& os, const THttpResponse& resp) { - resp.OutTo(os); + resp.OutTo(os); } diff --git a/library/cpp/http/server/response.h b/library/cpp/http/server/response.h index a75cb85605..7880babac5 100644 --- a/library/cpp/http/server/response.h +++ b/library/cpp/http/server/response.h @@ -1,8 +1,8 @@ #pragma once - + #include <library/cpp/http/misc/httpcodes.h> #include <library/cpp/http/io/stream.h> - + #include <util/generic/strbuf.h> #include <util/string/cast.h> @@ -12,23 +12,23 @@ class IOutputStream; class THttpResponse { public: THttpResponse() noexcept - : Code(HTTP_OK) - { - } - + : Code(HTTP_OK) + { + } + explicit THttpResponse(HttpCodes code) noexcept : Code(code) { } - template <typename ValueType> + template <typename ValueType> THttpResponse& AddHeader(const TString& name, const ValueType& value) { return AddHeader(THttpInputHeader(name, ToString(value))); } THttpResponse& AddHeader(const THttpInputHeader& header) { Headers.AddHeader(header); - + return *this; } @@ -38,8 +38,8 @@ public: return Headers; } - THttpResponse& SetContentType(const TStringBuf& contentType); - + THttpResponse& SetContentType(const TStringBuf& contentType); + /** * @note If @arg content isn't empty its size is automatically added as a * "Content-Length" header during output to IOutputStream. @@ -47,7 +47,7 @@ public: */ THttpResponse& SetContent(const TString& content) { Content = content; - + return *this; } @@ -61,8 +61,8 @@ public: * @see IOutputStream& operator << (IOutputStream&, const THttpResponse&) */ THttpResponse& SetContent(const TString& content, const TStringBuf& contentType) { - return SetContent(content).SetContentType(contentType); - } + return SetContent(content).SetContentType(contentType); + } HttpCodes HttpCode() const { return Code; diff --git a/library/cpp/http/server/response_ut.cpp b/library/cpp/http/server/response_ut.cpp index 73e2112ad3..ecd41015d9 100644 --- a/library/cpp/http/server/response_ut.cpp +++ b/library/cpp/http/server/response_ut.cpp @@ -2,11 +2,11 @@ #include <library/cpp/testing/unittest/registar.h> -#include <util/string/cast.h> - +#include <util/string/cast.h> + Y_UNIT_TEST_SUITE(TestHttpResponse) { Y_UNIT_TEST(TestCodeOnly) { - UNIT_ASSERT_STRINGS_EQUAL(ToString(THttpResponse()), "HTTP/1.1 200 Ok\r\n\r\n"); + UNIT_ASSERT_STRINGS_EQUAL(ToString(THttpResponse()), "HTTP/1.1 200 Ok\r\n\r\n"); UNIT_ASSERT_STRINGS_EQUAL(ToString(THttpResponse(HTTP_NOT_FOUND)), "HTTP/1.1 404 Not found\r\n\r\n"); } @@ -70,7 +70,7 @@ Y_UNIT_TEST_SUITE(TestHttpResponse) { "Content-Length: 10\r\n" "\r\n" "0123456789"; - UNIT_ASSERT_STRINGS_EQUAL(ToString(THttpResponse().SetContent("0123456789")), + UNIT_ASSERT_STRINGS_EQUAL(ToString(THttpResponse().SetContent("0123456789")), EXPECTED); } @@ -80,8 +80,8 @@ Y_UNIT_TEST_SUITE(TestHttpResponse) { "Content-Length: 28\r\n" "\r\n" "<xml><tag value=\"1\" /></xml>"; - THttpResponse resp; - resp.SetContent("<xml><tag value=\"1\" /></xml>").SetContentType("text/xml"); + THttpResponse resp; + resp.SetContent("<xml><tag value=\"1\" /></xml>").SetContentType("text/xml"); UNIT_ASSERT_STRINGS_EQUAL(ToString(resp), EXPECTED); } @@ -90,8 +90,8 @@ Y_UNIT_TEST_SUITE(TestHttpResponse) { resp.AddHeader(THttpInputHeader("X-Header-1", "ValueOne")) .AddHeader("X-Header-2", "ValueTwo") .AddHeader(THttpInputHeader("X-Header-3", "ValueThree")) - .SetContent("Some stuff") - .SetContentType("text/plain"); + .SetContent("Some stuff") + .SetContentType("text/plain"); THttpResponse copy = resp; UNIT_ASSERT_STRINGS_EQUAL(ToString(copy), ToString(resp)); @@ -102,19 +102,19 @@ Y_UNIT_TEST_SUITE(TestHttpResponse) { resp.AddHeader(THttpInputHeader("X-Header-1", "ValueOne")); resp.AddHeader(THttpInputHeader("X-Header-2", "ValueTwo")); resp.AddHeader(THttpInputHeader("X-Header-3", "ValueThree")); - resp.SetContent("Some stuff").SetContentType("text/plain"); + resp.SetContent("Some stuff").SetContentType("text/plain"); - THttpResponse copy; + THttpResponse copy; copy = resp; UNIT_ASSERT_STRINGS_EQUAL(ToString(copy), ToString(resp)); } Y_UNIT_TEST(TestEmptyContent) { - UNIT_ASSERT_STRINGS_EQUAL(ToString(THttpResponse().SetContent("")), "HTTP/1.1 200 Ok\r\n\r\n"); + UNIT_ASSERT_STRINGS_EQUAL(ToString(THttpResponse().SetContent("")), "HTTP/1.1 200 Ok\r\n\r\n"); } Y_UNIT_TEST(TestReturnReference) { - THttpResponse resp; + THttpResponse resp; UNIT_ASSERT_EQUAL(&resp, &resp.AddHeader("Header1", 1)); UNIT_ASSERT_EQUAL(&resp, &resp.AddHeader(THttpInputHeader("Header2", "2"))); @@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(TestHttpResponse) { UNIT_ASSERT_EQUAL(&resp, &resp.AddMultipleHeaders(headers)); UNIT_ASSERT_EQUAL(&resp, &resp.SetContent("some stuff")); - UNIT_ASSERT_EQUAL(&resp, &resp.SetContent("some other stuff").SetContentType("text/plain")); + UNIT_ASSERT_EQUAL(&resp, &resp.SetContent("some other stuff").SetContentType("text/plain")); } Y_UNIT_TEST(TestSetContentType) { diff --git a/library/cpp/http/server/ut/ya.make b/library/cpp/http/server/ut/ya.make index bcb4d4c0b8..17c20a8138 100644 --- a/library/cpp/http/server/ut/ya.make +++ b/library/cpp/http/server/ut/ya.make @@ -2,8 +2,8 @@ UNITTEST_FOR(library/cpp/http/server) OWNER(pg) -SIZE(MEDIUM) - +SIZE(MEDIUM) + SRCS( http_ut.cpp response_ut.cpp diff --git a/library/cpp/http/server/ya.make b/library/cpp/http/server/ya.make index bae6f33306..7b6bd59024 100644 --- a/library/cpp/http/server/ya.make +++ b/library/cpp/http/server/ya.make @@ -1,27 +1,27 @@ -LIBRARY() - -OWNER( - pg +LIBRARY() + +OWNER( + pg mvel kulikov g:base g:middle -) - -SRCS( - conn.cpp - http.cpp - http_ex.cpp - options.cpp +) + +SRCS( + conn.cpp + http.cpp + http_ex.cpp + options.cpp response.cpp -) - +) + PEERDIR( library/cpp/http/misc library/cpp/http/io library/cpp/threading/equeue ) -END() +END() RECURSE_FOR_TESTS(ut) diff --git a/library/cpp/hyperloglog/hyperloglog.cpp b/library/cpp/hyperloglog/hyperloglog.cpp index ec8352abe1..c162fad798 100644 --- a/library/cpp/hyperloglog/hyperloglog.cpp +++ b/library/cpp/hyperloglog/hyperloglog.cpp @@ -40,7 +40,7 @@ namespace { double EstimateBias(double e, unsigned precision) { static const TCorrection CORRECTIONS[1 + THyperLogLog::PRECISION_MAX - THyperLogLog::PRECISION_MIN] = { -#include "hyperloglog_corrections.inc" +#include "hyperloglog_corrections.inc" }; if (precision < THyperLogLog::PRECISION_MIN || precision > THyperLogLog::PRECISION_MAX) { return 0.; @@ -51,21 +51,21 @@ namespace { double GetThreshold(unsigned precision) { static const double THRESHOLD_DATA[1 + THyperLogLog::PRECISION_MAX - THyperLogLog::PRECISION_MIN] = { - 10, // Precision 4 - 20, // Precision 5 - 40, // Precision 6 - 80, // Precision 7 - 220, // Precision 8 - 400, // Precision 9 - 900, // Precision 10 - 1800, // Precision 11 - 3100, // Precision 12 - 6500, // Precision 13 - 11500, // Precision 14 - 20000, // Precision 15 - 50000, // Precision 16 - 120000, // Precision 17 - 350000 // Precision 18 + 10, // Precision 4 + 20, // Precision 5 + 40, // Precision 6 + 80, // Precision 7 + 220, // Precision 8 + 400, // Precision 9 + 900, // Precision 10 + 1800, // Precision 11 + 3100, // Precision 12 + 6500, // Precision 13 + 11500, // Precision 14 + 20000, // Precision 15 + 50000, // Precision 16 + 120000, // Precision 17 + 350000 // Precision 18 }; if (precision < THyperLogLog::PRECISION_MIN || precision > THyperLogLog::PRECISION_MAX) { return 0.; @@ -76,14 +76,14 @@ namespace { double EmpiricAlpha(size_t m) { switch (m) { - case 16: - return 0.673; - case 32: - return 0.697; - case 64: - return 0.709; - default: - return 0.7213 / (1.0 + 1.079 / m); + case 16: + return 0.673; + case 32: + return 0.697; + case 64: + return 0.709; + default: + return 0.7213 / (1.0 + 1.079 / m); } } @@ -109,7 +109,7 @@ void THyperLogLogBase::Update(ui64 hash) { const unsigned subHashBits = 8 * sizeof(hash) - Precision; const auto subHash = hash & MaskLowerBits(subHashBits); const auto leadingZeroes = subHash ? (subHashBits - GetValueBitCount(subHash)) : subHashBits; - const ui8 weight = static_cast<ui8>(leadingZeroes + 1); + const ui8 weight = static_cast<ui8>(leadingZeroes + 1); const size_t reg = static_cast<size_t>(hash >> subHashBits); RegistersRef[reg] = std::max(RegistersRef[reg], weight); diff --git a/library/cpp/hyperloglog/hyperloglog.h b/library/cpp/hyperloglog/hyperloglog.h index e79ee0ed77..d0a6e7376d 100644 --- a/library/cpp/hyperloglog/hyperloglog.h +++ b/library/cpp/hyperloglog/hyperloglog.h @@ -42,7 +42,7 @@ private: public: THyperLogLogWithAlloc(THyperLogLogWithAlloc&&) = default; - + THyperLogLogWithAlloc& operator=(THyperLogLogWithAlloc&&) = default; static THyperLogLogWithAlloc Create(unsigned precision) { diff --git a/library/cpp/hyperloglog/hyperloglog_ut.cpp b/library/cpp/hyperloglog/hyperloglog_ut.cpp index b987aa0fa4..7218536bce 100644 --- a/library/cpp/hyperloglog/hyperloglog_ut.cpp +++ b/library/cpp/hyperloglog/hyperloglog_ut.cpp @@ -15,15 +15,15 @@ Y_UNIT_TEST_SUITE(THyperLogLog) { auto counter = THyperLogLog::Create(18); static const std::pair<ui64, ui64> POINTS[] = { - {10, 10}, - {100, 100}, - {1000, 998}, - {10000, 9978}, - {100000, 99995}, - {1000000, 997017}, - {10000000, 9983891}, - {100000000, 100315572}, - {1000000000, 998791445}, + {10, 10}, + {100, 100}, + {1000, 998}, + {10000, 9978}, + {100000, 99995}, + {1000000, 997017}, + {10000000, 9983891}, + {100000000, 100315572}, + {1000000000, 998791445}, //1:37: {10000000000, 10015943904} }; ui64 unique = 0; diff --git a/library/cpp/int128/ut/int128_old_ut.cpp b/library/cpp/int128/ut/int128_old_ut.cpp index 2c5b9e9610..49cdcbcda3 100644 --- a/library/cpp/int128/ut/int128_old_ut.cpp +++ b/library/cpp/int128/ut/int128_old_ut.cpp @@ -5,19 +5,19 @@ class TUInt128Test: public TTestBase { UNIT_TEST_SUITE(TUInt128Test); - UNIT_TEST(Create); - UNIT_TEST(Minus); - UNIT_TEST(Plus); - UNIT_TEST(Shift) - UNIT_TEST(Overflow); - UNIT_TEST(Underflow); - UNIT_TEST(ToStringTest); - UNIT_TEST(FromStringTest); + UNIT_TEST(Create); + UNIT_TEST(Minus); + UNIT_TEST(Plus); + UNIT_TEST(Shift) + UNIT_TEST(Overflow); + UNIT_TEST(Underflow); + UNIT_TEST(ToStringTest); + UNIT_TEST(FromStringTest); #if defined(Y_HAVE_INT128) - UNIT_TEST(FromSystemUint128); + UNIT_TEST(FromSystemUint128); #endif UNIT_TEST_SUITE_END(); - + private: void Create(); void Minus(); diff --git a/library/cpp/ipv6_address/ipv6_address.cpp b/library/cpp/ipv6_address/ipv6_address.cpp index be8fcbae13..f34075e808 100644 --- a/library/cpp/ipv6_address/ipv6_address.cpp +++ b/library/cpp/ipv6_address/ipv6_address.cpp @@ -35,53 +35,53 @@ namespace { } } -void TIpv6Address::InitFrom(const in6_addr& addr) { - const ui64* const ui64Ptr = reinterpret_cast<const ui64*>(&addr); - const ui64 raw[2] = {SwapBytes(*ui64Ptr), SwapBytes(*(ui64Ptr + 1))}; +void TIpv6Address::InitFrom(const in6_addr& addr) { + const ui64* const ui64Ptr = reinterpret_cast<const ui64*>(&addr); + const ui64 raw[2] = {SwapBytes(*ui64Ptr), SwapBytes(*(ui64Ptr + 1))}; Ip = FromMemMSF(reinterpret_cast<const char*>(raw)); - Type_ = Ipv6; + Type_ = Ipv6; } -void TIpv6Address::InitFrom(const in_addr& addr) { +void TIpv6Address::InitFrom(const in_addr& addr) { unsigned long swapped = SwapBytes(addr.s_addr); Ip = ui128{0, swapped}; Type_ = Ipv4; } -void TIpv6Address::InitFrom(const sockaddr_in6& Addr) { +void TIpv6Address::InitFrom(const sockaddr_in6& Addr) { InitFrom(Addr.sin6_addr); ScopeId_ = Addr.sin6_scope_id; } -void TIpv6Address::InitFrom(const sockaddr_in& Addr) { +void TIpv6Address::InitFrom(const sockaddr_in& Addr) { InitFrom(Addr.sin_addr); } -TIpv6Address::TIpv6Address(const NAddr::IRemoteAddr& addr) { - if (addr.Addr()->sa_family == AF_INET) { // IPv4 - const sockaddr_in* Tmp = reinterpret_cast<const sockaddr_in*>(addr.Addr()); +TIpv6Address::TIpv6Address(const NAddr::IRemoteAddr& addr) { + if (addr.Addr()->sa_family == AF_INET) { // IPv4 + const sockaddr_in* Tmp = reinterpret_cast<const sockaddr_in*>(addr.Addr()); InitFrom(*Tmp); } else { // IPv6 - const sockaddr_in6* Tmp = reinterpret_cast<const sockaddr_in6*>(addr.Addr()); + const sockaddr_in6* Tmp = reinterpret_cast<const sockaddr_in6*>(addr.Addr()); InitFrom(*Tmp); } } -TIpv6Address::TIpv6Address(const sockaddr_in6& Addr) { +TIpv6Address::TIpv6Address(const sockaddr_in6& Addr) { InitFrom(Addr); } -TIpv6Address::TIpv6Address(const sockaddr_in& Addr) { +TIpv6Address::TIpv6Address(const sockaddr_in& Addr) { InitFrom(Addr); } -TIpv6Address::TIpv6Address(const in6_addr& addr, ui32 Scope) { +TIpv6Address::TIpv6Address(const in6_addr& addr, ui32 Scope) { InitFrom(addr); ScopeId_ = Scope; } -TIpv6Address::TIpv6Address(const in_addr& addr) { +TIpv6Address::TIpv6Address(const in_addr& addr) { InitFrom(addr); } TIpv6Address TIpv6Address::FromString(TStringBuf str, bool& ok) noexcept { const TIpType ipType = FigureOutType(str); - if (ipType == Ipv6) { + if (ipType == Ipv6) { ui32 scopeId = 0; if (size_t pos = str.find('%'); pos != TStringBuf::npos) { ::TryFromString(str.substr(pos + 1), scopeId); @@ -110,69 +110,69 @@ TIpv6Address TIpv6Address::FromString(TStringBuf str, bool& ok) noexcept { } } -TString TIpv6Address::ToString(bool* ok) const noexcept { +TString TIpv6Address::ToString(bool* ok) const noexcept { return ToString(true, ok); } -TString TIpv6Address::ToString(bool PrintScopeId, bool* ok) const noexcept { +TString TIpv6Address::ToString(bool PrintScopeId, bool* ok) const noexcept { TString result; - bool isOk = true; + bool isOk = true; - if (Type_ == TIpv6Address::Ipv4) { + if (Type_ == TIpv6Address::Ipv4) { result.resize(INET_ADDRSTRLEN + 2); in_addr addr; ToInAddr(addr); isOk = inet_ntop(AF_INET, &addr, result.begin(), INET_ADDRSTRLEN); result.resize(result.find('\0')); - } else if (Type_ == TIpv6Address::Ipv6) { + } else if (Type_ == TIpv6Address::Ipv6) { result.resize(INET6_ADDRSTRLEN + 2); in6_addr addr; ToIn6Addr(addr); isOk = inet_ntop(AF_INET6, &addr, result.begin(), INET6_ADDRSTRLEN); result.resize(result.find('\0')); - if (PrintScopeId) + if (PrintScopeId) result += "%" + ::ToString(ScopeId_); } else { result = "null"; - isOk = true; + isOk = true; } - if (ok) { + if (ok) { *ok = isOk; } return result; } -void TIpv6Address::ToSockaddrAndSocklen(sockaddr_in& sockAddrIPv4, - sockaddr_in6& sockAddrIPv6, // in - const sockaddr*& sockAddrPtr, - socklen_t& sockAddrSize, - ui16 Port) const { // out +void TIpv6Address::ToSockaddrAndSocklen(sockaddr_in& sockAddrIPv4, + sockaddr_in6& sockAddrIPv6, // in + const sockaddr*& sockAddrPtr, + socklen_t& sockAddrSize, + ui16 Port) const { // out - if (Type_ == Ipv4) { + if (Type_ == Ipv4) { memset(&sockAddrIPv4, 0, sizeof(sockAddrIPv4)); sockAddrIPv4.sin_family = AF_INET; - sockAddrIPv4.sin_port = htons(Port); + sockAddrIPv4.sin_port = htons(Port); ToInAddr(sockAddrIPv4.sin_addr); sockAddrSize = sizeof(sockAddrIPv4); - sockAddrPtr = reinterpret_cast<sockaddr*>(&sockAddrIPv4); + sockAddrPtr = reinterpret_cast<sockaddr*>(&sockAddrIPv4); - } else if (Type_ == Ipv6) { + } else if (Type_ == Ipv6) { memset(&sockAddrIPv6, 0, sizeof(sockAddrIPv6)); sockAddrIPv6.sin6_family = AF_INET6; - sockAddrIPv6.sin6_port = htons(Port); + sockAddrIPv6.sin6_port = htons(Port); ToIn6Addr(sockAddrIPv6.sin6_addr); sockAddrIPv6.sin6_scope_id = ScopeId_; sockAddrIPv6.sin6_flowinfo = 0; sockAddrSize = sizeof(sockAddrIPv6); - sockAddrPtr = reinterpret_cast<sockaddr*>(&sockAddrIPv6); + sockAddrPtr = reinterpret_cast<sockaddr*>(&sockAddrIPv6); } else Y_VERIFY(false); } -void TIpv6Address::ToInAddr(in_addr& Addr4) const { +void TIpv6Address::ToInAddr(in_addr& Addr4) const { Y_VERIFY(Type_ == TIpv6Address::Ipv4); Zero(Addr4); @@ -181,22 +181,22 @@ void TIpv6Address::ToInAddr(in_addr& Addr4) const { Y_VERIFY(GetHigh(Ip) == 0, " "); Addr4.s_addr = SwapBytes(Value); } -void TIpv6Address::ToIn6Addr(in6_addr& Addr6) const { +void TIpv6Address::ToIn6Addr(in6_addr& Addr6) const { Y_VERIFY(Type_ == TIpv6Address::Ipv6); Zero(Addr6); ui64 Raw[2] = {GetHigh(Ip), GetLow(Ip)}; - *Raw = SwapBytes(*Raw); - Raw[1] = SwapBytes(1 [Raw]); + *Raw = SwapBytes(*Raw); + Raw[1] = SwapBytes(1 [Raw]); memcpy(&Addr6, Raw, sizeof(Raw)); } -void TIpv6Address::Save(IOutputStream* out) const { +void TIpv6Address::Save(IOutputStream* out) const { ::Save(out, Ip); ::Save(out, static_cast<ui8>(Type_)); ::Save(out, ScopeId_); } -void TIpv6Address::Load(IInputStream* in) { +void TIpv6Address::Load(IInputStream* in) { ::Load(in, Ip); ui8 num; ::Load(in, num); @@ -210,21 +210,21 @@ bool TIpv6Address::Isv4MappedTov6() const noexcept { /// the IPv4-mapped IPv6 addresses. In these addresses, the first 80 bits are zero, the next 16 bits are one, /// and the remaining 32 bits are the IPv4 address. - if (Type_ != Ipv6) + if (Type_ != Ipv6) return false; if (GetHigh(Ip) != 0) return false; // First 64 bit are not zero -> it is not ipv4-mapped-ipv6 address const ui64 Low = GetLow(Ip) >> 32; - if (Low != 0x0000ffff) + if (Low != 0x0000ffff) return false; return true; } TIpv6Address TIpv6Address::TryToExtractIpv4From6() const noexcept { - if (Isv4MappedTov6() == false) + if (Isv4MappedTov6() == false) return TIpv6Address(); const ui64 NewLow = GetLow(Ip) & 0x00000000ffffffff; @@ -233,7 +233,7 @@ TIpv6Address TIpv6Address::TryToExtractIpv4From6() const noexcept { } TIpv6Address TIpv6Address::Normalized() const noexcept { - if (Isv4MappedTov6() == false) + if (Isv4MappedTov6() == false) return *this; TIpv6Address Result = TryToExtractIpv4From6(); @@ -241,7 +241,7 @@ TIpv6Address TIpv6Address::Normalized() const noexcept { return Result; } -IOutputStream& operator<<(IOutputStream& Out, const TIpv6Address::TIpType Type) noexcept { +IOutputStream& operator<<(IOutputStream& Out, const TIpv6Address::TIpType Type) noexcept { switch (Type) { case TIpv6Address::Ipv4: Out << "Ipv4"; @@ -255,10 +255,10 @@ IOutputStream& operator<<(IOutputStream& Out, const TIpv6Address::TIpType Type) } } -IOutputStream& operator<<(IOutputStream& out, const TIpv6Address& ipv6Address) noexcept { - bool ok; - const TString& strIp = ipv6Address.ToString(&ok); - if (!ok) { +IOutputStream& operator<<(IOutputStream& out, const TIpv6Address& ipv6Address) noexcept { + bool ok; + const TString& strIp = ipv6Address.ToString(&ok); + if (!ok) { return out << "Can not convert ip to string"; } else { return out << strIp; @@ -271,29 +271,29 @@ TString THostAddressAndPort::ToString() const noexcept { return Str.Str(); } -IOutputStream& operator<<(IOutputStream& Out, const THostAddressAndPort& HostAddressAndPort) noexcept { +IOutputStream& operator<<(IOutputStream& Out, const THostAddressAndPort& HostAddressAndPort) noexcept { Out << HostAddressAndPort.Ip << ":" << HostAddressAndPort.Port; return Out; } namespace { - class TRemoteAddr: public NAddr::IRemoteAddr { + class TRemoteAddr: public NAddr::IRemoteAddr { public: - TRemoteAddr(const TIpv6Address& Address, TIpPort Port); - const sockaddr* Addr() const override; - socklen_t Len() const override; + TRemoteAddr(const TIpv6Address& Address, TIpPort Port); + const sockaddr* Addr() const override; + socklen_t Len() const override; private: - sockaddr_in SockAddrIPv4; - sockaddr_in6 SockAddrIPv6; - const sockaddr* SockAddrPtr = nullptr; - socklen_t SockAddrSize = 0; + sockaddr_in SockAddrIPv4; + sockaddr_in6 SockAddrIPv6; + const sockaddr* SockAddrPtr = nullptr; + socklen_t SockAddrSize = 0; }; - TRemoteAddr::TRemoteAddr(const TIpv6Address& Address, TIpPort Port) { + TRemoteAddr::TRemoteAddr(const TIpv6Address& Address, TIpPort Port) { Address.ToSockaddrAndSocklen(SockAddrIPv4, SockAddrIPv6, SockAddrPtr, SockAddrSize, Port); } - const sockaddr* TRemoteAddr::Addr() const { + const sockaddr* TRemoteAddr::Addr() const { return SockAddrPtr; } socklen_t TRemoteAddr::Len() const { @@ -301,13 +301,13 @@ namespace { } } -NAddr::IRemoteAddr* ToIRemoteAddr(const TIpv6Address& Address, TIpPort Port) { +NAddr::IRemoteAddr* ToIRemoteAddr(const TIpv6Address& Address, TIpPort Port) { return new TRemoteAddr(Address, Port); } -std::tuple<THostAddressAndPort, TString, TIpPort> ParseHostAndMayBePortFromString(const TString& RawStr, - TIpPort DefaultPort, - bool& Ok) noexcept { +std::tuple<THostAddressAndPort, TString, TIpPort> ParseHostAndMayBePortFromString(const TString& RawStr, + TIpPort DefaultPort, + bool& Ok) noexcept { // Cout << "ParseHostAndMayBePortFromString: " << RawStr << ", Port: " << DefaultPort << Endl; using TRes = std::tuple<THostAddressAndPort, TString, TIpPort>; @@ -315,7 +315,7 @@ std::tuple<THostAddressAndPort, TString, TIpPort> ParseHostAndMayBePortFromStrin // --------------------------------------------------------------------- const size_t BracketColPos = RawStr.find("]:"); - if (BracketColPos != TString::npos) { + if (BracketColPos != TString::npos) { // [ipv6]:port if (!RawStr.StartsWith('[')) { Ok = false; @@ -324,13 +324,13 @@ std::tuple<THostAddressAndPort, TString, TIpPort> ParseHostAndMayBePortFromStrin const TStringBuf StrIpv6(RawStr.begin() + 1, RawStr.begin() + BracketColPos); const TStringBuf StrPort(RawStr.begin() + BracketColPos + 2, RawStr.end()); - bool IpConverted; + bool IpConverted; const TIpv6Address Ip = TIpv6Address::FromString(StrIpv6, IpConverted); if (!IpConverted) { Ok = false; return {}; } - if (Ip.Type() != TIpv6Address::Ipv6) { + if (Ip.Type() != TIpv6Address::Ipv6) { Ok = false; return {}; } @@ -347,7 +347,7 @@ std::tuple<THostAddressAndPort, TString, TIpPort> ParseHostAndMayBePortFromStrin // --------------------------------------------------------------------- - if (RawStr.StartsWith('[')) { + if (RawStr.StartsWith('[')) { // [ipv6] if (!RawStr.EndsWith(']')) { Ok = false; @@ -355,7 +355,7 @@ std::tuple<THostAddressAndPort, TString, TIpPort> ParseHostAndMayBePortFromStrin } const TStringBuf StrIpv6(RawStr.begin() + 1, RawStr.end() - 1); - bool IpConverted; + bool IpConverted; const TIpv6Address Ip = TIpv6Address::FromString(StrIpv6, IpConverted); if (!IpConverted || Ip.Type() != TIpv6Address::Ipv6) { Ok = false; @@ -370,15 +370,15 @@ std::tuple<THostAddressAndPort, TString, TIpPort> ParseHostAndMayBePortFromStrin // --------------------------------------------------------------------- const size_t ColPos = RawStr.find(':'); - if (ColPos != TString::npos) { + if (ColPos != TString::npos) { // host:port // ipv4:port // ipv6 { - bool IpConverted; + bool IpConverted; const TIpv6Address Ipv6 = TIpv6Address::FromString(RawStr, IpConverted); - if (IpConverted && Ipv6.Type() == TIpv6Address::Ipv6) { + if (IpConverted && Ipv6.Type() == TIpv6Address::Ipv6) { // ipv6 Ok = true; TRes Res{{Ipv6, DefaultPort}, {}, {}}; @@ -395,9 +395,9 @@ std::tuple<THostAddressAndPort, TString, TIpPort> ParseHostAndMayBePortFromStrin const TStringBuf StrIpv4OrHost(RawStr.begin(), RawStr.begin() + ColPos); { - bool IpConverted; + bool IpConverted; const TIpv6Address Ipv4 = TIpv6Address::FromString(StrIpv4OrHost, IpConverted); - if (IpConverted && Ipv4.Type() == TIpv6Address::Ipv4) { + if (IpConverted && Ipv4.Type() == TIpv6Address::Ipv4) { // ipv4:port Ok = true; TRes Res{{Ipv4, Port}, {}, {}}; @@ -417,9 +417,9 @@ std::tuple<THostAddressAndPort, TString, TIpPort> ParseHostAndMayBePortFromStrin { // ipv4 - bool IpConverted; + bool IpConverted; const TIpv6Address Ipv4 = TIpv6Address::FromString(RawStr, IpConverted); - if (IpConverted && Ipv4.Type() == TIpv6Address::Ipv4) { + if (IpConverted && Ipv4.Type() == TIpv6Address::Ipv4) { Ok = true; TRes Res{{Ipv4, DefaultPort}, {}, {}}; return Res; diff --git a/library/cpp/ipv6_address/ipv6_address.h b/library/cpp/ipv6_address/ipv6_address.h index 1d7eb0b65f..bb6cbc6229 100644 --- a/library/cpp/ipv6_address/ipv6_address.h +++ b/library/cpp/ipv6_address/ipv6_address.h @@ -28,9 +28,9 @@ struct sockaddr_in6; // TODO (dimanne): rename to something like TIntInetAddress or THostAddress class TIpv6Address { public: - enum TIpType { Ipv6, - Ipv4, - LAST }; + enum TIpType { Ipv6, + Ipv4, + LAST }; constexpr TIpv6Address() noexcept = default; constexpr TIpv6Address(const TIpv6Address&) noexcept = default; @@ -54,11 +54,11 @@ public: Ip = {hi, lo}; } - explicit TIpv6Address(const NAddr::IRemoteAddr& addr); - explicit TIpv6Address(const sockaddr_in6& Addr); - explicit TIpv6Address(const sockaddr_in& Addr); - explicit TIpv6Address(const in6_addr& addr, ui32 Scope); - explicit TIpv6Address(const in_addr& addr); + explicit TIpv6Address(const NAddr::IRemoteAddr& addr); + explicit TIpv6Address(const sockaddr_in6& Addr); + explicit TIpv6Address(const sockaddr_in& Addr); + explicit TIpv6Address(const in6_addr& addr, ui32 Scope); + explicit TIpv6Address(const in_addr& addr); static TIpv6Address FromString(TStringBuf srcStr, bool& ok) noexcept; @@ -78,31 +78,31 @@ public: return !IsValid(); } - bool Isv4MappedTov6() const noexcept; + bool Isv4MappedTov6() const noexcept; TIpv6Address TryToExtractIpv4From6() const noexcept; TIpv6Address Normalized() const noexcept; - TString ToString(bool* ok = nullptr) const noexcept; - TString ToString(bool PrintScopeId, bool* ok = nullptr) const noexcept; + TString ToString(bool* ok = nullptr) const noexcept; + TString ToString(bool PrintScopeId, bool* ok = nullptr) const noexcept; - void ToSockaddrAndSocklen(sockaddr_in& sockAddrIPv4, - sockaddr_in6& sockAddrIPv6, // in - const sockaddr*& sockAddrPtr, - socklen_t& sockAddrSize, - ui16 Port) const; // out + void ToSockaddrAndSocklen(sockaddr_in& sockAddrIPv4, + sockaddr_in6& sockAddrIPv6, // in + const sockaddr*& sockAddrPtr, + socklen_t& sockAddrSize, + ui16 Port) const; // out - void ToInAddr(in_addr& Addr4) const; - void ToIn6Addr(in6_addr& Addr6) const; + void ToInAddr(in_addr& Addr4) const; + void ToIn6Addr(in6_addr& Addr6) const; // int SocketFamily() const; constexpr bool operator<(const TIpv6Address& other) const noexcept { - if (Type_ != other.Type_) + if (Type_ != other.Type_) return Type_ > other.Type_; else return Ip < other.Ip; } constexpr bool operator>(const TIpv6Address& other) const noexcept { - if (Type_ != other.Type_) + if (Type_ != other.Type_) return Type_ < other.Type_; else return Ip > other.Ip; @@ -137,22 +137,22 @@ public: return ScopeId_; } - void Save(IOutputStream* out) const; - void Load(IInputStream* in); + void Save(IOutputStream* out) const; + void Load(IInputStream* in); private: - void InitFrom(const in6_addr& addr); - void InitFrom(const in_addr& addr); + void InitFrom(const in6_addr& addr); + void InitFrom(const in_addr& addr); - void InitFrom(const sockaddr_in6& Addr); - void InitFrom(const sockaddr_in& Addr); + void InitFrom(const sockaddr_in6& Addr); + void InitFrom(const sockaddr_in& Addr); ui128 Ip{}; - TIpType Type_ = LAST; - ui32 ScopeId_ = 0; + TIpType Type_ = LAST; + ui32 ScopeId_ = 0; }; -IOutputStream& operator<<(IOutputStream& Out, const TIpv6Address::TIpType Type) noexcept; -IOutputStream& operator<<(IOutputStream& Out, const TIpv6Address& ipv6Address) noexcept; +IOutputStream& operator<<(IOutputStream& Out, const TIpv6Address::TIpType Type) noexcept; +IOutputStream& operator<<(IOutputStream& Out, const TIpv6Address& ipv6Address) noexcept; constexpr TIpv6Address Get127001() noexcept { return {127, 0, 0, 1}; @@ -184,7 +184,7 @@ struct THostAddressAndPort { TIpv6Address Ip {}; TIpPort Port {0}; }; -IOutputStream& operator<<(IOutputStream& Out, const THostAddressAndPort& HostAddressAndPort) noexcept; +IOutputStream& operator<<(IOutputStream& Out, const THostAddressAndPort& HostAddressAndPort) noexcept; /// /// Returns @@ -199,22 +199,22 @@ IOutputStream& operator<<(IOutputStream& Out, const THostAddressAndPort& HostAdd /// 2001::7348 // port wil be equal to DefaultPort /// [2001::7348]:80 /// -std::tuple<THostAddressAndPort, TString, TIpPort> ParseHostAndMayBePortFromString(const TString& RawStr, - TIpPort DefaultPort, - bool& Ok) noexcept; +std::tuple<THostAddressAndPort, TString, TIpPort> ParseHostAndMayBePortFromString(const TString& RawStr, + TIpPort DefaultPort, + bool& Ok) noexcept; using TIpv6AddressesSet = THashSet<TIpv6Address>; template <> struct THash<TIpv6Address> { - inline size_t operator()(const TIpv6Address& ip) const { + inline size_t operator()(const TIpv6Address& ip) const { const ui128& Tmp = static_cast<ui128>(ip); return CombineHashes(THash<ui128>()(Tmp), THash<ui8>()(static_cast<ui8>(ip.Type()))); } }; template <> struct THash<THostAddressAndPort> { - inline size_t operator()(const THostAddressAndPort& IpAndPort) const { + inline size_t operator()(const THostAddressAndPort& IpAndPort) const { return CombineHashes(THash<TIpv6Address>()(IpAndPort.Ip), THash<TIpPort>()(IpAndPort.Port)); } }; @@ -222,13 +222,13 @@ struct THash<THostAddressAndPort> { namespace std { template <> struct hash<TIpv6Address> { - std::size_t operator()(const TIpv6Address& Ip) const noexcept { + std::size_t operator()(const TIpv6Address& Ip) const noexcept { return THash<TIpv6Address>()(Ip); } }; } -NAddr::IRemoteAddr* ToIRemoteAddr(const TIpv6Address& Address, TIpPort Port); +NAddr::IRemoteAddr* ToIRemoteAddr(const TIpv6Address& Address, TIpPort Port); // template <> // class TSerializer<TIpv6Address> { diff --git a/library/cpp/ipv6_address/ut/ipv6_address_ut.cpp b/library/cpp/ipv6_address/ut/ipv6_address_ut.cpp index 73bb7cffdd..377ffd0f65 100644 --- a/library/cpp/ipv6_address/ut/ipv6_address_ut.cpp +++ b/library/cpp/ipv6_address/ut/ipv6_address_ut.cpp @@ -2,7 +2,7 @@ #include <library/cpp/ipv6_address/ipv6_address.h> #include <unordered_set> -class TIpv6AddressTest: public TTestBase { +class TIpv6AddressTest: public TTestBase { UNIT_TEST_SUITE(TIpv6AddressTest); UNIT_TEST(ParseHostAndMayBePortFromString_data); UNIT_TEST(CheckAddressValidity) @@ -19,7 +19,7 @@ UNIT_TEST_SUITE_REGISTRATION(TIpv6AddressTest); using TResult = std::tuple<THostAddressAndPort, TString, TIpPort>; TResult IpRes(TString Ip, TIpPort Port) { - bool Ok; + bool Ok; THostAddressAndPort HostAddressAndPort; HostAddressAndPort.Ip = TIpv6Address::FromString(Ip, Ok); Y_VERIFY(Ok); @@ -30,24 +30,24 @@ TResult HostRes(TString HostName, TIpPort Port) { return TResult({}, HostName, Port); } -void ParseHostAndMayBePortFromString(TString RawStr, - TIpPort DefaultPort, +void ParseHostAndMayBePortFromString(TString RawStr, + TIpPort DefaultPort, const TResult ExpectedResult, - const bool ExpectedOk) { - bool Ok = false; + const bool ExpectedOk) { + bool Ok = false; const TResult ActualResult = ParseHostAndMayBePortFromString(RawStr, DefaultPort, Ok); UNIT_ASSERT(Ok == ExpectedOk); - if (Ok == false) + if (Ok == false) return; UNIT_ASSERT(ActualResult == ExpectedResult); } -void CheckIpDefPortAgainstIpPortDefPort_v4OrHost(TString Ip, - TIpPort Port, +void CheckIpDefPortAgainstIpPortDefPort_v4OrHost(TString Ip, + TIpPort Port, const TResult ExpectedResult, - const bool ExpectedOk) { + const bool ExpectedOk) { ParseHostAndMayBePortFromString(Ip, Port, ExpectedResult, ExpectedOk); TString New = Ip + ":" + ToString(Port); @@ -62,7 +62,7 @@ void CheckIpDefPortAgainstIpPortDefPort_v6(TString Ip, TIpPort Port, const TResu } void CheckIpDefPortAgainstIpPortDefPort(TString Ip, TIpPort Port, const TResult ExpectedResult, const bool ExpectedOk) { - if (Ip.find(':') == TString::npos) { + if (Ip.find(':') == TString::npos) { CheckIpDefPortAgainstIpPortDefPort_v4OrHost(Ip, Port, ExpectedResult, ExpectedOk); } else { CheckIpDefPortAgainstIpPortDefPort_v6(Ip, Port, ExpectedResult, ExpectedOk); diff --git a/library/cpp/json/common/defs.cpp b/library/cpp/json/common/defs.cpp index da86da82e4..7ae904ba54 100644 --- a/library/cpp/json/common/defs.cpp +++ b/library/cpp/json/common/defs.cpp @@ -1,68 +1,68 @@ -#include "defs.h" - -using namespace NJson; - -TJsonCallbacks::~TJsonCallbacks() { -} - -bool TJsonCallbacks::OnNull() { - return true; -} - -bool TJsonCallbacks::OnBoolean(bool) { - return true; -} - -bool TJsonCallbacks::OnInteger(long long) { - return true; -} - -bool TJsonCallbacks::OnUInteger(unsigned long long) { - return true; -} - -bool TJsonCallbacks::OnDouble(double) { - return true; -} - -bool TJsonCallbacks::OnString(const TStringBuf&) { - return true; -} - -bool TJsonCallbacks::OnOpenMap() { - return true; -} - -bool TJsonCallbacks::OnMapKey(const TStringBuf&) { - return true; -} - -bool TJsonCallbacks::OnCloseMap() { - return true; -} - -bool TJsonCallbacks::OnOpenArray() { - return true; -} - -bool TJsonCallbacks::OnCloseArray() { - return true; -} - -bool TJsonCallbacks::OnStringNoCopy(const TStringBuf& s) { - return OnString(s); -} - -bool TJsonCallbacks::OnMapKeyNoCopy(const TStringBuf& s) { - return OnMapKey(s); -} - -bool TJsonCallbacks::OnEnd() { - return true; -} - -void TJsonCallbacks::OnError(size_t off, TStringBuf reason) { - if (ThrowException) { - ythrow TJsonException() << "JSON error at offset " << off << " (" << reason << ")"; - } -} +#include "defs.h" + +using namespace NJson; + +TJsonCallbacks::~TJsonCallbacks() { +} + +bool TJsonCallbacks::OnNull() { + return true; +} + +bool TJsonCallbacks::OnBoolean(bool) { + return true; +} + +bool TJsonCallbacks::OnInteger(long long) { + return true; +} + +bool TJsonCallbacks::OnUInteger(unsigned long long) { + return true; +} + +bool TJsonCallbacks::OnDouble(double) { + return true; +} + +bool TJsonCallbacks::OnString(const TStringBuf&) { + return true; +} + +bool TJsonCallbacks::OnOpenMap() { + return true; +} + +bool TJsonCallbacks::OnMapKey(const TStringBuf&) { + return true; +} + +bool TJsonCallbacks::OnCloseMap() { + return true; +} + +bool TJsonCallbacks::OnOpenArray() { + return true; +} + +bool TJsonCallbacks::OnCloseArray() { + return true; +} + +bool TJsonCallbacks::OnStringNoCopy(const TStringBuf& s) { + return OnString(s); +} + +bool TJsonCallbacks::OnMapKeyNoCopy(const TStringBuf& s) { + return OnMapKey(s); +} + +bool TJsonCallbacks::OnEnd() { + return true; +} + +void TJsonCallbacks::OnError(size_t off, TStringBuf reason) { + if (ThrowException) { + ythrow TJsonException() << "JSON error at offset " << off << " (" << reason << ")"; + } +} diff --git a/library/cpp/json/common/defs.h b/library/cpp/json/common/defs.h index d3c8761bcc..8e0e6923fc 100644 --- a/library/cpp/json/common/defs.h +++ b/library/cpp/json/common/defs.h @@ -1,38 +1,38 @@ -#pragma once - -#include <util/generic/strbuf.h> -#include <util/generic/yexception.h> - -namespace NJson { - class TJsonException: public yexception { - }; - - class TJsonCallbacks { - public: - explicit TJsonCallbacks(bool throwException = false) - : ThrowException(throwException) - { - } - - virtual ~TJsonCallbacks(); - - virtual bool OnNull(); - virtual bool OnBoolean(bool); - virtual bool OnInteger(long long); - virtual bool OnUInteger(unsigned long long); - virtual bool OnDouble(double); - virtual bool OnString(const TStringBuf&); - virtual bool OnOpenMap(); - virtual bool OnMapKey(const TStringBuf&); - virtual bool OnCloseMap(); - virtual bool OnOpenArray(); - virtual bool OnCloseArray(); - virtual bool OnStringNoCopy(const TStringBuf& s); - virtual bool OnMapKeyNoCopy(const TStringBuf& s); - virtual bool OnEnd(); - virtual void OnError(size_t off, TStringBuf reason); - - protected: - bool ThrowException; - }; -} +#pragma once + +#include <util/generic/strbuf.h> +#include <util/generic/yexception.h> + +namespace NJson { + class TJsonException: public yexception { + }; + + class TJsonCallbacks { + public: + explicit TJsonCallbacks(bool throwException = false) + : ThrowException(throwException) + { + } + + virtual ~TJsonCallbacks(); + + virtual bool OnNull(); + virtual bool OnBoolean(bool); + virtual bool OnInteger(long long); + virtual bool OnUInteger(unsigned long long); + virtual bool OnDouble(double); + virtual bool OnString(const TStringBuf&); + virtual bool OnOpenMap(); + virtual bool OnMapKey(const TStringBuf&); + virtual bool OnCloseMap(); + virtual bool OnOpenArray(); + virtual bool OnCloseArray(); + virtual bool OnStringNoCopy(const TStringBuf& s); + virtual bool OnMapKeyNoCopy(const TStringBuf& s); + virtual bool OnEnd(); + virtual void OnError(size_t off, TStringBuf reason); + + protected: + bool ThrowException; + }; +} diff --git a/library/cpp/json/common/ya.make b/library/cpp/json/common/ya.make index 5bbd3b0792..c64976ccd1 100644 --- a/library/cpp/json/common/ya.make +++ b/library/cpp/json/common/ya.make @@ -1,9 +1,9 @@ -LIBRARY() - -OWNER(pg) - -SRCS( - defs.cpp -) - -END() +LIBRARY() + +OWNER(pg) + +SRCS( + defs.cpp +) + +END() diff --git a/library/cpp/json/domscheme_traits.h b/library/cpp/json/domscheme_traits.h index a5a99cd8cf..68b149a690 100644 --- a/library/cpp/json/domscheme_traits.h +++ b/library/cpp/json/domscheme_traits.h @@ -1,7 +1,7 @@ #pragma once -#include "json_value.h" -#include "json_reader.h" +#include "json_value.h" +#include "json_reader.h" #include "json_writer.h" #include <util/generic/algorithm.h> @@ -125,7 +125,7 @@ struct TJsonTraits { // boolean ops static inline void Get(TConstValueRef v, bool def, bool& b) { b = - v->GetType() == NJson::JSON_UNDEFINED ? def : v->IsNull() ? def : v->GetBooleanRobust(); + v->GetType() == NJson::JSON_UNDEFINED ? def : v->IsNull() ? def : v->GetBooleanRobust(); } static inline void Get(TConstValueRef v, bool& b) { @@ -136,14 +136,14 @@ struct TJsonTraits { return v->IsBoolean(); } -#define INTEGER_OPS(type, checkOp, getOp) \ - static inline void Get(TConstValueRef v, type def, type& i) { \ - i = v->checkOp() ? v->getOp() : def; \ - } \ - static inline void Get(TConstValueRef v, type& i) { \ - i = v->getOp(); \ - } \ - static inline bool IsValidPrimitive(const type&, TConstValueRef v) { \ +#define INTEGER_OPS(type, checkOp, getOp) \ + static inline void Get(TConstValueRef v, type def, type& i) { \ + i = v->checkOp() ? v->getOp() : def; \ + } \ + static inline void Get(TConstValueRef v, type& i) { \ + i = v->getOp(); \ + } \ + static inline bool IsValidPrimitive(const type&, TConstValueRef v) { \ return v->checkOp() && v->getOp() >= Min<type>() && v->getOp() <= Max<type>(); \ } diff --git a/library/cpp/json/easy_parse/json_easy_parser.cpp b/library/cpp/json/easy_parse/json_easy_parser.cpp index 3c781f544b..4856768635 100644 --- a/library/cpp/json/easy_parse/json_easy_parser.cpp +++ b/library/cpp/json/easy_parse/json_easy_parser.cpp @@ -5,232 +5,232 @@ #include <util/string/strip.h> namespace NJson { - static TString MAP_IDENTIFIER = "{}"; - static TString ARRAY_IDENTIFIER = "[]"; - static TString ANY_IDENTIFIER = "*"; - - static void ParsePath(TString path, TVector<TPathElem>* res) { - TVector<const char*> parts; - Split(path.begin(), '/', &parts); - for (size_t n = 0; n < parts.size(); ++n) { + static TString MAP_IDENTIFIER = "{}"; + static TString ARRAY_IDENTIFIER = "[]"; + static TString ANY_IDENTIFIER = "*"; + + static void ParsePath(TString path, TVector<TPathElem>* res) { + TVector<const char*> parts; + Split(path.begin(), '/', &parts); + for (size_t n = 0; n < parts.size(); ++n) { TString part = Strip(parts[n]); - if (!part.empty()) { - if (part[0] != '[') { - res->push_back(TPathElem(NImpl::MAP)); - res->push_back(TPathElem(part)); - } else { - int arrayCounter; - try { - arrayCounter = FromString<int>(part.substr(1, part.length() - 2)); - } catch (yexception&) { - arrayCounter = -1; - } - res->push_back(TPathElem(arrayCounter)); + if (!part.empty()) { + if (part[0] != '[') { + res->push_back(TPathElem(NImpl::MAP)); + res->push_back(TPathElem(part)); + } else { + int arrayCounter; + try { + arrayCounter = FromString<int>(part.substr(1, part.length() - 2)); + } catch (yexception&) { + arrayCounter = -1; + } + res->push_back(TPathElem(arrayCounter)); } } } } - void TJsonParser::AddField(const TString& path, bool nonEmpty) { - Fields.emplace_back(); - Fields.back().NonEmpty = nonEmpty; - ParsePath(path, &Fields.back().Path); - } - - TString TJsonParser::ConvertToTabDelimited(const TString& json) const { - TStringInput in(json); - TStringStream out; - ConvertToTabDelimited(in, out); - return out.Str(); - } - - class TRewriteJsonImpl: public NJson::TJsonCallbacks { - const TJsonParser& Parent; - TVector<TString> FieldValues; - TVector<TPathElem> Stack; - bool ShouldUpdateOnArrayChange; - int CurrentFieldIdx; - bool HasFormatError; - - private: - static bool PathElementMatch(const TPathElem& templ, const TPathElem& real) { - if (templ.Type != real.Type) - return false; - if (templ.Type == NImpl::ARRAY) - return templ.ArrayCounter == -1 || templ.ArrayCounter == real.ArrayCounter; - if (templ.Type == NImpl::MAP_KEY) - return templ.Key == ANY_IDENTIFIER || templ.Key == real.Key; - return true; - } - - bool CheckFilter(const TVector<TPathElem>& path) const { - if (Stack.size() < path.size()) + void TJsonParser::AddField(const TString& path, bool nonEmpty) { + Fields.emplace_back(); + Fields.back().NonEmpty = nonEmpty; + ParsePath(path, &Fields.back().Path); + } + + TString TJsonParser::ConvertToTabDelimited(const TString& json) const { + TStringInput in(json); + TStringStream out; + ConvertToTabDelimited(in, out); + return out.Str(); + } + + class TRewriteJsonImpl: public NJson::TJsonCallbacks { + const TJsonParser& Parent; + TVector<TString> FieldValues; + TVector<TPathElem> Stack; + bool ShouldUpdateOnArrayChange; + int CurrentFieldIdx; + bool HasFormatError; + + private: + static bool PathElementMatch(const TPathElem& templ, const TPathElem& real) { + if (templ.Type != real.Type) + return false; + if (templ.Type == NImpl::ARRAY) + return templ.ArrayCounter == -1 || templ.ArrayCounter == real.ArrayCounter; + if (templ.Type == NImpl::MAP_KEY) + return templ.Key == ANY_IDENTIFIER || templ.Key == real.Key; + return true; + } + + bool CheckFilter(const TVector<TPathElem>& path) const { + if (Stack.size() < path.size()) return false; for (size_t n = 0; n < path.size(); ++n) { - if (!PathElementMatch(path[n], Stack[n])) - return false; - } - return true; + if (!PathElementMatch(path[n], Stack[n])) + return false; + } + return true; } - void UpdateRule() { + void UpdateRule() { for (size_t n = 0; n < Parent.Fields.size(); ++n) { - if (FieldValues[n].empty() && CheckFilter(Parent.Fields[n].Path)) { - CurrentFieldIdx = n; - return; - } - } - CurrentFieldIdx = -1; - } - - void Pop() { - Stack.pop_back(); - } - - void IncreaseArrayCounter() { - if (!Stack.empty() && Stack.back().Type == NImpl::ARRAY) { - ++Stack.back().ArrayCounter; - if (ShouldUpdateOnArrayChange) - UpdateRule(); + if (FieldValues[n].empty() && CheckFilter(Parent.Fields[n].Path)) { + CurrentFieldIdx = n; + return; + } } - } - - template <class T> - bool OnValue(const T& val) { - IncreaseArrayCounter(); - if (CurrentFieldIdx >= 0) { - FieldValues[CurrentFieldIdx] = ToString(val); + CurrentFieldIdx = -1; + } + + void Pop() { + Stack.pop_back(); + } + + void IncreaseArrayCounter() { + if (!Stack.empty() && Stack.back().Type == NImpl::ARRAY) { + ++Stack.back().ArrayCounter; + if (ShouldUpdateOnArrayChange) + UpdateRule(); + } + } + + template <class T> + bool OnValue(const T& val) { + IncreaseArrayCounter(); + if (CurrentFieldIdx >= 0) { + FieldValues[CurrentFieldIdx] = ToString(val); UpdateRule(); - } - return true; - } - - public: - TRewriteJsonImpl(const TJsonParser& parent) - : Parent(parent) - , FieldValues(parent.Fields.size()) - , ShouldUpdateOnArrayChange(false) - , CurrentFieldIdx(-1) - , HasFormatError(false) - { + } + return true; + } + + public: + TRewriteJsonImpl(const TJsonParser& parent) + : Parent(parent) + , FieldValues(parent.Fields.size()) + , ShouldUpdateOnArrayChange(false) + , CurrentFieldIdx(-1) + , HasFormatError(false) + { for (size_t n = 0; n < Parent.Fields.size(); ++n) { if (!Parent.Fields[n].Path.empty() && Parent.Fields[n].Path.back().Type == NImpl::ARRAY) - ShouldUpdateOnArrayChange = true; - } - } - - bool OnOpenMap() override { - IncreaseArrayCounter(); - Stack.push_back(TPathElem(NImpl::MAP)); - if (CurrentFieldIdx >= 0) - HasFormatError = true; - else - UpdateRule(); - return true; - } - - bool OnOpenArray() override { - IncreaseArrayCounter(); - Stack.push_back(TPathElem(-1)); - if (CurrentFieldIdx >= 0) - HasFormatError = true; - else - UpdateRule(); - return true; - } - - bool OnCloseMap() override { - while (!Stack.empty() && Stack.back().Type != NImpl::MAP) - Pop(); - if (!Stack.empty()) - Pop(); + ShouldUpdateOnArrayChange = true; + } + } + + bool OnOpenMap() override { + IncreaseArrayCounter(); + Stack.push_back(TPathElem(NImpl::MAP)); + if (CurrentFieldIdx >= 0) + HasFormatError = true; + else + UpdateRule(); + return true; + } + + bool OnOpenArray() override { + IncreaseArrayCounter(); + Stack.push_back(TPathElem(-1)); + if (CurrentFieldIdx >= 0) + HasFormatError = true; + else + UpdateRule(); + return true; + } + + bool OnCloseMap() override { + while (!Stack.empty() && Stack.back().Type != NImpl::MAP) + Pop(); + if (!Stack.empty()) + Pop(); UpdateRule(); - return true; - } + return true; + } - bool OnCloseArray() override { - if (!Stack.empty()) - Pop(); + bool OnCloseArray() override { + if (!Stack.empty()) + Pop(); UpdateRule(); - return true; - } - - bool OnMapKey(const TStringBuf& key) override { - if (!Stack.empty() && Stack.back().Type == NImpl::MAP_KEY) { - Pop(); - UpdateRule(); - } + return true; + } + + bool OnMapKey(const TStringBuf& key) override { + if (!Stack.empty() && Stack.back().Type == NImpl::MAP_KEY) { + Pop(); + UpdateRule(); + } Stack.push_back(TPathElem(TString{key})); - if (CurrentFieldIdx >= 0) - HasFormatError = true; - else - UpdateRule(); - return true; - } + if (CurrentFieldIdx >= 0) + HasFormatError = true; + else + UpdateRule(); + return true; + } - bool OnBoolean(bool b) override { - return OnValue(b); - } + bool OnBoolean(bool b) override { + return OnValue(b); + } - bool OnInteger(long long i) override { - return OnValue(i); + bool OnInteger(long long i) override { + return OnValue(i); } - bool OnDouble(double f) override { - return OnValue(f); - } + bool OnDouble(double f) override { + return OnValue(f); + } - bool OnString(const TStringBuf& str) override { - return OnValue(str); - } + bool OnString(const TStringBuf& str) override { + return OnValue(str); + } - bool IsOK() const { - if (HasFormatError) - return false; + bool IsOK() const { + if (HasFormatError) + return false; for (size_t n = 0; n < FieldValues.size(); ++n) - if (Parent.Fields[n].NonEmpty && FieldValues[n].empty()) - return false; - return true; - } + if (Parent.Fields[n].NonEmpty && FieldValues[n].empty()) + return false; + return true; + } - void WriteTo(IOutputStream& out) const { + void WriteTo(IOutputStream& out) const { for (size_t n = 0; n < FieldValues.size(); ++n) - out << "\t" << FieldValues[n]; - } - - void WriteTo(TVector<TString>* res) const { - *res = FieldValues; - } - }; - - void TJsonParser::ConvertToTabDelimited(IInputStream& in, IOutputStream& out) const { - TRewriteJsonImpl impl(*this); - ReadJson(&in, &impl); - if (impl.IsOK()) { - out << Prefix; - impl.WriteTo(out); - out.Flush(); - } + out << "\t" << FieldValues[n]; + } + + void WriteTo(TVector<TString>* res) const { + *res = FieldValues; + } + }; + + void TJsonParser::ConvertToTabDelimited(IInputStream& in, IOutputStream& out) const { + TRewriteJsonImpl impl(*this); + ReadJson(&in, &impl); + if (impl.IsOK()) { + out << Prefix; + impl.WriteTo(out); + out.Flush(); + } } - bool TJsonParser::Parse(const TString& json, TVector<TString>* res) const { - TRewriteJsonImpl impl(*this); - TStringInput in(json); - ReadJson(&in, &impl); - if (impl.IsOK()) { - impl.WriteTo(res); - return true; - } else - return false; + bool TJsonParser::Parse(const TString& json, TVector<TString>* res) const { + TRewriteJsonImpl impl(*this); + TStringInput in(json); + ReadJson(&in, &impl); + if (impl.IsOK()) { + impl.WriteTo(res); + return true; + } else + return false; } - //struct TTestMe { - // TTestMe() { - // TJsonParser worker; - // worker.AddField("/x/y/z", true); - // TString ret1 = worker.ConvertToTabDelimited("{ \"x\" : { \"y\" : { \"w\" : 1, \"z\" : 2 } } }"); - // TString ret2 = worker.ConvertToTabDelimited(" [1, 2, 3, 4, 5] "); - // } - //} testMe; + //struct TTestMe { + // TTestMe() { + // TJsonParser worker; + // worker.AddField("/x/y/z", true); + // TString ret1 = worker.ConvertToTabDelimited("{ \"x\" : { \"y\" : { \"w\" : 1, \"z\" : 2 } } }"); + // TString ret2 = worker.ConvertToTabDelimited(" [1, 2, 3, 4, 5] "); + // } + //} testMe; } diff --git a/library/cpp/json/easy_parse/json_easy_parser.h b/library/cpp/json/easy_parse/json_easy_parser.h index 59d7791ab1..310ec6f2ec 100644 --- a/library/cpp/json/easy_parse/json_easy_parser.h +++ b/library/cpp/json/easy_parse/json_easy_parser.h @@ -1,5 +1,5 @@ #pragma once - + #include <util/generic/string.h> #include <util/generic/vector.h> #include <util/stream/input.h> @@ -33,14 +33,14 @@ namespace NJson { friend class TRewriteJsonImpl; - void ConvertToTabDelimited(IInputStream& in, IOutputStream& out) const; - + void ConvertToTabDelimited(IInputStream& in, IOutputStream& out) const; + public: - void SetPrefix(const TString& prefix) { + void SetPrefix(const TString& prefix) { Prefix = prefix; } - void AddField(const TString& path, bool mustExist); - TString ConvertToTabDelimited(const TString& json) const; - bool Parse(const TString& json, TVector<TString>* res) const; + void AddField(const TString& path, bool mustExist); + TString ConvertToTabDelimited(const TString& json) const; + bool Parse(const TString& json, TVector<TString>* res) const; }; } diff --git a/library/cpp/json/easy_parse/json_easy_parser_impl.h b/library/cpp/json/easy_parse/json_easy_parser_impl.h index ec55d838b3..4ac14e8a44 100644 --- a/library/cpp/json/easy_parse/json_easy_parser_impl.h +++ b/library/cpp/json/easy_parse/json_easy_parser_impl.h @@ -22,7 +22,7 @@ namespace NJson { { } - TPathElemImpl(const TStringType& key) + TPathElemImpl(const TStringType& key) : Type(NImpl::MAP_KEY) , Key(key) , ArrayCounter() @@ -31,8 +31,8 @@ namespace NJson { TPathElemImpl(int arrayCounter) : Type(NImpl::ARRAY) - , ArrayCounter(arrayCounter) - { + , ArrayCounter(arrayCounter) + { } }; diff --git a/library/cpp/json/fast_sax/parser.h b/library/cpp/json/fast_sax/parser.h index b5f031dd9e..31f5348da9 100644 --- a/library/cpp/json/fast_sax/parser.h +++ b/library/cpp/json/fast_sax/parser.h @@ -1,13 +1,13 @@ -#pragma once - +#pragma once + #include <library/cpp/json/common/defs.h> - -namespace NJson { - bool ReadJsonFast(TStringBuf in, TJsonCallbacks* callbacks); - - inline bool ValidateJsonFast(TStringBuf in, bool throwOnError = false) { + +namespace NJson { + bool ReadJsonFast(TStringBuf in, TJsonCallbacks* callbacks); + + inline bool ValidateJsonFast(TStringBuf in, bool throwOnError = false) { Y_ASSERT(false); // this method is broken, see details in IGNIETFERRO-1243. Use NJson::ValidateJson instead, or fix this one before using - TJsonCallbacks c(throwOnError); - return ReadJsonFast(in, &c); - } -} + TJsonCallbacks c(throwOnError); + return ReadJsonFast(in, &c); + } +} diff --git a/library/cpp/json/fast_sax/parser.rl6 b/library/cpp/json/fast_sax/parser.rl6 index edb4e9ee1b..0f1ad3b3b1 100644 --- a/library/cpp/json/fast_sax/parser.rl6 +++ b/library/cpp/json/fast_sax/parser.rl6 @@ -2,7 +2,7 @@ #include <library/cpp/json/fast_sax/parser.h> #include <util/string/cast.h> -#include <util/generic/buffer.h> +#include <util/generic/buffer.h> #include <util/generic/strbuf.h> #include <util/generic/ymath.h> @@ -15,7 +15,7 @@ enum EStoredStr { struct TParserCtx { TJsonCallbacks& Hndl; - TBuffer Buffer; + TBuffer Buffer; TStringBuf String; EStoredStr Stored = SS_NONE; bool ExpectValue = true; @@ -142,9 +142,9 @@ struct TParserCtx { } bool OnStrE() { - Buffer.Clear(); - Buffer.Reserve(2 * (te - ts)); - + Buffer.Clear(); + Buffer.Reserve(2 * (te - ts)); + return OnString(UnescapeJsonUnicode(TStringBuf(ts + 1, te - ts - 2), Buffer.data()), SS_MUSTCOPY); } @@ -170,8 +170,8 @@ struct TParserCtx { Stored = SS_NONE; switch (stored) { - default: - return false; + default: + return false; case SS_NOCOPY: return Hndl.OnMapKeyNoCopy(String); case SS_MUSTCOPY: diff --git a/library/cpp/json/fast_sax/unescape.cpp b/library/cpp/json/fast_sax/unescape.cpp index 72109b0b5e..772fe92a0d 100644 --- a/library/cpp/json/fast_sax/unescape.cpp +++ b/library/cpp/json/fast_sax/unescape.cpp @@ -1,7 +1,7 @@ -#include "unescape.h" - -#include <util/string/escape.h> - -TStringBuf UnescapeJsonUnicode(TStringBuf data, char* scratch) { +#include "unescape.h" + +#include <util/string/escape.h> + +TStringBuf UnescapeJsonUnicode(TStringBuf data, char* scratch) { return TStringBuf(scratch, UnescapeC(data.data(), data.size(), scratch)); -} +} diff --git a/library/cpp/json/fast_sax/unescape.h b/library/cpp/json/fast_sax/unescape.h index 5e40e1e866..55315548bc 100644 --- a/library/cpp/json/fast_sax/unescape.h +++ b/library/cpp/json/fast_sax/unescape.h @@ -1,5 +1,5 @@ -#pragma once - -#include <util/generic/strbuf.h> - -TStringBuf UnescapeJsonUnicode(TStringBuf data, char* scratch); +#pragma once + +#include <util/generic/strbuf.h> + +TStringBuf UnescapeJsonUnicode(TStringBuf data, char* scratch); diff --git a/library/cpp/json/fast_sax/ya.make b/library/cpp/json/fast_sax/ya.make index c6447ab6ac..46e5c7c5d9 100644 --- a/library/cpp/json/fast_sax/ya.make +++ b/library/cpp/json/fast_sax/ya.make @@ -1,17 +1,17 @@ -LIBRARY() - +LIBRARY() + OWNER( pg velavokr ) - -PEERDIR( + +PEERDIR( library/cpp/json/common -) - -SRCS( - parser.rl6 - unescape.cpp -) - -END() +) + +SRCS( + parser.rl6 + unescape.cpp +) + +END() diff --git a/library/cpp/json/flex_buffers/cvt.cpp b/library/cpp/json/flex_buffers/cvt.cpp index fee0cea0b8..9f7d630373 100644 --- a/library/cpp/json/flex_buffers/cvt.cpp +++ b/library/cpp/json/flex_buffers/cvt.cpp @@ -1,139 +1,139 @@ -#include "cvt.h" - -#include <flatbuffers/flexbuffers.h> - +#include "cvt.h" + +#include <flatbuffers/flexbuffers.h> + #include <library/cpp/json/fast_sax/parser.h> #include <library/cpp/json/json_reader.h> - -#include <util/generic/vector.h> -#include <util/stream/output.h> -#include <util/stream/input.h> -#include <util/memory/pool.h> - -using namespace NJson; - -namespace { - struct TJsonToFlexCallbacks: public TJsonCallbacks { - inline TJsonToFlexCallbacks() - : P(8192) - { - } - + +#include <util/generic/vector.h> +#include <util/stream/output.h> +#include <util/stream/input.h> +#include <util/memory/pool.h> + +using namespace NJson; + +namespace { + struct TJsonToFlexCallbacks: public TJsonCallbacks { + inline TJsonToFlexCallbacks() + : P(8192) + { + } + bool OnNull() override { - B.Null(); - - return true; - } - + B.Null(); + + return true; + } + bool OnBoolean(bool v) override { - B.Bool(v); - - return true; - } - + B.Bool(v); + + return true; + } + bool OnInteger(long long v) override { - B.Int(v); - - return true; - } - + B.Int(v); + + return true; + } + bool OnUInteger(unsigned long long v) override { - B.UInt(v); - - return true; - } - + B.UInt(v); + + return true; + } + bool OnDouble(double v) override { - B.Double(v); - - return true; - } - + B.Double(v); + + return true; + } + bool OnString(const TStringBuf& v) override { B.String(v.data(), v.size()); - - return true; - } - + + return true; + } + bool OnOpenMap() override { - S.push_back(B.StartMap()); - - return true; - } - + S.push_back(B.StartMap()); + + return true; + } + bool OnMapKey(const TStringBuf& v) override { auto iv = P.AppendCString(v); - + B.Key(iv.data(), iv.size()); - - return true; - } - + + return true; + } + bool OnCloseMap() override { - B.EndMap(PopOffset()); - - return true; - } - + B.EndMap(PopOffset()); + + return true; + } + bool OnOpenArray() override { - S.push_back(B.StartVector()); - - return true; - } - + S.push_back(B.StartVector()); + + return true; + } + bool OnCloseArray() override { - B.EndVector(PopOffset(), false, false); - - return true; - } - + B.EndVector(PopOffset(), false, false); + + return true; + } + bool OnStringNoCopy(const TStringBuf& s) override { - return OnString(s); - } - + return OnString(s); + } + bool OnMapKeyNoCopy(const TStringBuf& s) override { - return OnMapKey(s); - } - + return OnMapKey(s); + } + bool OnEnd() override { - B.Finish(); - - Y_ENSURE(S.empty()); - - return true; - } - + B.Finish(); + + Y_ENSURE(S.empty()); + + return true; + } + void OnError(size_t, TStringBuf reason) override { - ythrow yexception() << reason; - } - - inline size_t PopOffset() { - auto res = S.back(); - - S.pop_back(); - - return res; - } - - inline auto& Buffer() { - return B.GetBuffer(); - } - - flexbuffers::Builder B; - TVector<size_t> S; - TMemoryPool P; - }; -} - -void NJson::ConvertJsonToFlexBuffers(TStringBuf input, TFlexBuffersData& result) { - TJsonToFlexCallbacks cb; - - ReadJsonFast(input, &cb); - result.swap(const_cast<std::vector<ui8>&>(cb.Buffer())); -} - -TString NJson::FlexToString(const TFlexBuffersData& v) { + ythrow yexception() << reason; + } + + inline size_t PopOffset() { + auto res = S.back(); + + S.pop_back(); + + return res; + } + + inline auto& Buffer() { + return B.GetBuffer(); + } + + flexbuffers::Builder B; + TVector<size_t> S; + TMemoryPool P; + }; +} + +void NJson::ConvertJsonToFlexBuffers(TStringBuf input, TFlexBuffersData& result) { + TJsonToFlexCallbacks cb; + + ReadJsonFast(input, &cb); + result.swap(const_cast<std::vector<ui8>&>(cb.Buffer())); +} + +TString NJson::FlexToString(const TFlexBuffersData& v) { auto root = flexbuffers::GetRoot(v.data(), v.size()); - - return TString(root.ToString()); -} + + return TString(root.ToString()); +} diff --git a/library/cpp/json/flex_buffers/cvt.h b/library/cpp/json/flex_buffers/cvt.h index 82d2874268..489f2dad83 100644 --- a/library/cpp/json/flex_buffers/cvt.h +++ b/library/cpp/json/flex_buffers/cvt.h @@ -1,20 +1,20 @@ -#pragma once - -#include <util/generic/vector.h> -#include <util/generic/strbuf.h> -#include <util/generic/string.h> - -namespace NJson { - using TFlexBuffersData = TVector<ui8>; - - TString FlexToString(const TFlexBuffersData& v); - void ConvertJsonToFlexBuffers(TStringBuf input, TFlexBuffersData& result); - - inline TFlexBuffersData ConvertJsonToFlexBuffers(TStringBuf input) { - TFlexBuffersData result; - - ConvertJsonToFlexBuffers(input, result); - - return result; - } -} +#pragma once + +#include <util/generic/vector.h> +#include <util/generic/strbuf.h> +#include <util/generic/string.h> + +namespace NJson { + using TFlexBuffersData = TVector<ui8>; + + TString FlexToString(const TFlexBuffersData& v); + void ConvertJsonToFlexBuffers(TStringBuf input, TFlexBuffersData& result); + + inline TFlexBuffersData ConvertJsonToFlexBuffers(TStringBuf input) { + TFlexBuffersData result; + + ConvertJsonToFlexBuffers(input, result); + + return result; + } +} diff --git a/library/cpp/json/flex_buffers/ut/cvt_ut.cpp b/library/cpp/json/flex_buffers/ut/cvt_ut.cpp index 9fffef4d38..de3eb0d520 100644 --- a/library/cpp/json/flex_buffers/ut/cvt_ut.cpp +++ b/library/cpp/json/flex_buffers/ut/cvt_ut.cpp @@ -1,21 +1,21 @@ #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/json/flex_buffers/cvt.h> - -using namespace NJson; - -static auto JSON = R"({ - "a": { - "b": [1, 2, 3], - "c": ["x", "y", 3, "z"] - } -})"; - -static auto RES = R"({ a: { b: [ 1, 2, 3 ], c: [ "x", "y", 3, "z" ] } })"; - -Y_UNIT_TEST_SUITE(JsonToFlex) { - Y_UNIT_TEST(Test1) { - auto buf = ConvertJsonToFlexBuffers(JSON); - - UNIT_ASSERT_VALUES_EQUAL(FlexToString(buf), RES); - } -} + +using namespace NJson; + +static auto JSON = R"({ + "a": { + "b": [1, 2, 3], + "c": ["x", "y", 3, "z"] + } +})"; + +static auto RES = R"({ a: { b: [ 1, 2, 3 ], c: [ "x", "y", 3, "z" ] } })"; + +Y_UNIT_TEST_SUITE(JsonToFlex) { + Y_UNIT_TEST(Test1) { + auto buf = ConvertJsonToFlexBuffers(JSON); + + UNIT_ASSERT_VALUES_EQUAL(FlexToString(buf), RES); + } +} diff --git a/library/cpp/json/flex_buffers/ut/ya.make b/library/cpp/json/flex_buffers/ut/ya.make index 3fdc93f88e..148ac13856 100644 --- a/library/cpp/json/flex_buffers/ut/ya.make +++ b/library/cpp/json/flex_buffers/ut/ya.make @@ -1,9 +1,9 @@ UNITTEST_FOR(library/cpp/json/flex_buffers) - -OWNER(pg) - -SRCS( - cvt_ut.cpp -) - -END() + +OWNER(pg) + +SRCS( + cvt_ut.cpp +) + +END() diff --git a/library/cpp/json/flex_buffers/ya.make b/library/cpp/json/flex_buffers/ya.make index 3ece5e3703..1b38b2f3d1 100644 --- a/library/cpp/json/flex_buffers/ya.make +++ b/library/cpp/json/flex_buffers/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - -OWNER(pg) - +LIBRARY() + +OWNER(pg) + ADDINCL(contrib/libs/flatbuffers/include) - -PEERDIR( + +PEERDIR( library/cpp/json - contrib/libs/flatbuffers -) - -SRCS( - cvt.cpp -) - -END() + contrib/libs/flatbuffers +) + +SRCS( + cvt.cpp +) + +END() diff --git a/library/cpp/json/fuzzy_test/main.cpp b/library/cpp/json/fuzzy_test/main.cpp index 29a53aac14..bdbf2679ff 100644 --- a/library/cpp/json/fuzzy_test/main.cpp +++ b/library/cpp/json/fuzzy_test/main.cpp @@ -1,30 +1,30 @@ #include <library/cpp/json/json_reader.h> - -#include <util/random/random.h> -#include <util/stream/str.h> - -extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { + +#include <util/random/random.h> +#include <util/stream/str.h> + +extern "C" int LLVMFuzzerTestOneInput(const ui8* data, size_t size) { const auto json = TString((const char*)data, size); - - try { - NJson::TJsonValue value; - NJson::ReadJsonFastTree(json, &value, true); - } catch (...) { - //Cout << json << " -> " << CurrentExceptionMessage() << Endl; - } - - try { - NJson::TJsonCallbacks cb; - NJson::ReadJsonFast(json, &cb); - } catch (...) { - //Cout << json << " -> " << CurrentExceptionMessage() << Endl; - } - - try { - NJson::ValidateJson(json); - } catch (...) { - //Cout << json << " -> " << CurrentExceptionMessage() << Endl; - } - - return 0; -} + + try { + NJson::TJsonValue value; + NJson::ReadJsonFastTree(json, &value, true); + } catch (...) { + //Cout << json << " -> " << CurrentExceptionMessage() << Endl; + } + + try { + NJson::TJsonCallbacks cb; + NJson::ReadJsonFast(json, &cb); + } catch (...) { + //Cout << json << " -> " << CurrentExceptionMessage() << Endl; + } + + try { + NJson::ValidateJson(json); + } catch (...) { + //Cout << json << " -> " << CurrentExceptionMessage() << Endl; + } + + return 0; +} diff --git a/library/cpp/json/fuzzy_test/ya.make b/library/cpp/json/fuzzy_test/ya.make index ff50bc1f62..1ccd587546 100644 --- a/library/cpp/json/fuzzy_test/ya.make +++ b/library/cpp/json/fuzzy_test/ya.make @@ -1,13 +1,13 @@ FUZZ() - -OWNER(pg) - -PEERDIR( + +OWNER(pg) + +PEERDIR( library/cpp/json -) - -SRCS( - main.cpp -) - -END() +) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/json/json_prettifier.cpp b/library/cpp/json/json_prettifier.cpp index bb16aab44e..cf36ae210d 100644 --- a/library/cpp/json/json_prettifier.cpp +++ b/library/cpp/json/json_prettifier.cpp @@ -1,277 +1,277 @@ #include "json_prettifier.h" #include <util/generic/deque.h> -#include <util/generic/algorithm.h> +#include <util/generic/algorithm.h> #include <util/memory/pool.h> #include <util/string/util.h> #include <library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h> namespace NJson { - struct TRewritableOut { - IOutputStream& Slave; - - char Last = 0; - bool Dirty = false; - - TRewritableOut(IOutputStream& sl) - : Slave(sl) - { - } - - template <typename T> - void Write(const T& t) { - Flush(); - Slave << t; - } - - void Hold(char c) { - if (Dirty) - Flush(); - Last = c; - Dirty = true; - } - - void Flush() { - if (Dirty) { - Slave << Last; - Dirty = false; - } - } - - void Revert() { + struct TRewritableOut { + IOutputStream& Slave; + + char Last = 0; + bool Dirty = false; + + TRewritableOut(IOutputStream& sl) + : Slave(sl) + { + } + + template <typename T> + void Write(const T& t) { + Flush(); + Slave << t; + } + + void Hold(char c) { + if (Dirty) + Flush(); + Last = c; + Dirty = true; + } + + void Flush() { + if (Dirty) { + Slave << Last; + Dirty = false; + } + } + + void Revert() { Dirty = false; } - }; - - struct TSpaces { - char S[256]; - - TSpaces() { - memset(&S, ' ', sizeof(S)); - } - - TStringBuf Get(ui8 sz) const { - return TStringBuf(S, sz); - } - }; - - bool TJsonPrettifier::MayUnquoteNew(TStringBuf s) { - static str_spn alpha("a-zA-Z_@$", true); - static str_spn alnum("a-zA-Z_@$0-9.-", true); - static TStringBuf true0("true"); - static TStringBuf false0("false"); - static TStringBuf null0("null"); - - return !!s && alpha.chars_table[(ui8)s[0]] && alnum.cbrk(s.begin() + 1, s.end()) == s.end() && !EqualToOneOf(s, null0, true0, false0); + }; + + struct TSpaces { + char S[256]; + + TSpaces() { + memset(&S, ' ', sizeof(S)); + } + + TStringBuf Get(ui8 sz) const { + return TStringBuf(S, sz); + } + }; + + bool TJsonPrettifier::MayUnquoteNew(TStringBuf s) { + static str_spn alpha("a-zA-Z_@$", true); + static str_spn alnum("a-zA-Z_@$0-9.-", true); + static TStringBuf true0("true"); + static TStringBuf false0("false"); + static TStringBuf null0("null"); + + return !!s && alpha.chars_table[(ui8)s[0]] && alnum.cbrk(s.begin() + 1, s.end()) == s.end() && !EqualToOneOf(s, null0, true0, false0); } - // to keep arcadia tests happy - bool TJsonPrettifier::MayUnquoteOld(TStringBuf s) { - static str_spn alpha("a-zA-Z_@$", true); - static str_spn alnum("a-zA-Z_@$0-9", true); - static TStringBuf true0("true"); - static TStringBuf false0("false"); - static TStringBuf true1("on"); - static TStringBuf false1("off"); - static TStringBuf true2("da"); - static TStringBuf false2("net"); - static TStringBuf null0("null"); - - return !!s && alpha.chars_table[(ui8)s[0]] && alnum.cbrk(s.begin() + 1, s.end()) == s.end() && !EqualToOneOf(s, null0, true0, false0, true1, false1, true2, false2); - } - - class TPrettifier: public TJsonCallbacks { - TRewritableOut Out; - TStringBuf Spaces; - TStringBuf Quote; - TStringBuf Unsafe; - TStringBuf Safe; - - ui32 Level = 0; - ui32 MaxPaddingLevel; - - bool Unquote = false; - bool Compactify = false; - bool NewUnquote = false; - - public: - TPrettifier(IOutputStream& out, const TJsonPrettifier& p) - : Out(out) - , MaxPaddingLevel(p.MaxPaddingLevel) - , Unquote(p.Unquote) - , Compactify(p.Compactify) - , NewUnquote(p.NewUnquote) - { - static TSpaces spaces; - Spaces = spaces.Get(p.Padding); - if (p.SingleQuotes) { - Quote = Unsafe = "'"; - Safe = "\""; - } else { - Quote = Unsafe = "\""; - Safe = "'"; - } + // to keep arcadia tests happy + bool TJsonPrettifier::MayUnquoteOld(TStringBuf s) { + static str_spn alpha("a-zA-Z_@$", true); + static str_spn alnum("a-zA-Z_@$0-9", true); + static TStringBuf true0("true"); + static TStringBuf false0("false"); + static TStringBuf true1("on"); + static TStringBuf false1("off"); + static TStringBuf true2("da"); + static TStringBuf false2("net"); + static TStringBuf null0("null"); + + return !!s && alpha.chars_table[(ui8)s[0]] && alnum.cbrk(s.begin() + 1, s.end()) == s.end() && !EqualToOneOf(s, null0, true0, false0, true1, false1, true2, false2); + } + + class TPrettifier: public TJsonCallbacks { + TRewritableOut Out; + TStringBuf Spaces; + TStringBuf Quote; + TStringBuf Unsafe; + TStringBuf Safe; + + ui32 Level = 0; + ui32 MaxPaddingLevel; + + bool Unquote = false; + bool Compactify = false; + bool NewUnquote = false; + + public: + TPrettifier(IOutputStream& out, const TJsonPrettifier& p) + : Out(out) + , MaxPaddingLevel(p.MaxPaddingLevel) + , Unquote(p.Unquote) + , Compactify(p.Compactify) + , NewUnquote(p.NewUnquote) + { + static TSpaces spaces; + Spaces = spaces.Get(p.Padding); + if (p.SingleQuotes) { + Quote = Unsafe = "'"; + Safe = "\""; + } else { + Quote = Unsafe = "\""; + Safe = "'"; + } + } + + void Pad(bool close = false) { + if (Compactify) { + Out.Flush(); + return; + } + if (Level > MaxPaddingLevel || (Level == MaxPaddingLevel && close)) { + Out.Write(" "); + return; + } + if (Level || close) { + Out.Write(Spaces ? "\n" : " "); + } + for (ui32 i = 0; i < Level; ++i) { + Out.Write(Spaces); + } + } + + void WriteSpace(char sp) { + if (Compactify) { + Out.Flush(); + return; + } + + Out.Write(sp); } - void Pad(bool close = false) { - if (Compactify) { - Out.Flush(); - return; - } - if (Level > MaxPaddingLevel || (Level == MaxPaddingLevel && close)) { - Out.Write(" "); - return; - } - if (Level || close) { - Out.Write(Spaces ? "\n" : " "); - } - for (ui32 i = 0; i < Level; ++i) { - Out.Write(Spaces); - } + void OnVal() { + if (Out.Dirty && ':' == Out.Last) { + WriteSpace(' '); + } else { + Pad(); + } } - - void WriteSpace(char sp) { - if (Compactify) { - Out.Flush(); - return; - } - - Out.Write(sp); - } - - void OnVal() { - if (Out.Dirty && ':' == Out.Last) { - WriteSpace(' '); - } else { - Pad(); - } + + void AfterVal() { + Out.Hold(','); } - - void AfterVal() { - Out.Hold(','); - } - - template <typename T> - bool WriteVal(const T& t) { - OnVal(); - Out.Write(t); - AfterVal(); - return true; + + template <typename T> + bool WriteVal(const T& t) { + OnVal(); + Out.Write(t); + AfterVal(); + return true; } - - bool OnNull() override { + + bool OnNull() override { return WriteVal(TStringBuf("null")); } - bool OnBoolean(bool v) override { + bool OnBoolean(bool v) override { return WriteVal(v ? TStringBuf("true") : TStringBuf("false")); } - bool OnInteger(long long i) override { - return WriteVal(i); - } - - bool OnUInteger(unsigned long long i) override { - return WriteVal(i); - } - - bool OnDouble(double d) override { - return WriteVal(d); - } - - void WriteString(TStringBuf s) { - if (Unquote && (NewUnquote ? TJsonPrettifier::MayUnquoteNew(s) : TJsonPrettifier::MayUnquoteOld(s))) { - Out.Slave << s; - } else { - Out.Slave << Quote; - NEscJ::EscapeJ<false, true>(s, Out.Slave, Safe, Unsafe); - Out.Slave << Quote; - } - } - - bool OnString(const TStringBuf& s) override { - OnVal(); - WriteString(s); - AfterVal(); - return true; - } - - bool OnOpen(char c) { - OnVal(); - Level++; - Out.Hold(c); - return true; - } - - bool OnOpenMap() override { - return OnOpen('{'); - } - - bool OnOpenArray() override { - return OnOpen('['); - } - - bool OnMapKey(const TStringBuf& k) override { - OnVal(); - WriteString(k); - WriteSpace(' '); - Out.Hold(':'); - return true; - } - - bool OnClose(char c) { - if (!Level) - return false; - - Level--; - - if (Out.Dirty && c == Out.Last) { - WriteSpace(' '); - } else { - Out.Revert(); - Pad(true); - } - - return true; - } - - bool OnCloseMap() override { - if (!OnClose('{')) - return false; - Out.Write("}"); - AfterVal(); - return true; - } + bool OnInteger(long long i) override { + return WriteVal(i); + } - bool OnCloseArray() override { - if (!OnClose('[')) - return false; - Out.Write("]"); - AfterVal(); - return true; + bool OnUInteger(unsigned long long i) override { + return WriteVal(i); } - bool OnEnd() override { - return !Level; + bool OnDouble(double d) override { + return WriteVal(d); + } + + void WriteString(TStringBuf s) { + if (Unquote && (NewUnquote ? TJsonPrettifier::MayUnquoteNew(s) : TJsonPrettifier::MayUnquoteOld(s))) { + Out.Slave << s; + } else { + Out.Slave << Quote; + NEscJ::EscapeJ<false, true>(s, Out.Slave, Safe, Unsafe); + Out.Slave << Quote; + } + } + + bool OnString(const TStringBuf& s) override { + OnVal(); + WriteString(s); + AfterVal(); + return true; + } + + bool OnOpen(char c) { + OnVal(); + Level++; + Out.Hold(c); + return true; + } + + bool OnOpenMap() override { + return OnOpen('{'); + } + + bool OnOpenArray() override { + return OnOpen('['); + } + + bool OnMapKey(const TStringBuf& k) override { + OnVal(); + WriteString(k); + WriteSpace(' '); + Out.Hold(':'); + return true; } - }; - bool TJsonPrettifier::Prettify(TStringBuf in, IOutputStream& out) const { - TPrettifier p(out, *this); - if (Strict) { + bool OnClose(char c) { + if (!Level) + return false; + + Level--; + + if (Out.Dirty && c == Out.Last) { + WriteSpace(' '); + } else { + Out.Revert(); + Pad(true); + } + + return true; + } + + bool OnCloseMap() override { + if (!OnClose('{')) + return false; + Out.Write("}"); + AfterVal(); + return true; + } + + bool OnCloseArray() override { + if (!OnClose('[')) + return false; + Out.Write("]"); + AfterVal(); + return true; + } + + bool OnEnd() override { + return !Level; + } + }; + + bool TJsonPrettifier::Prettify(TStringBuf in, IOutputStream& out) const { + TPrettifier p(out, *this); + if (Strict) { TMemoryInput mIn(in.data(), in.size()); - return ReadJson(&mIn, &p); + return ReadJson(&mIn, &p); } else { - return ReadJsonFast(in, &p); + return ReadJsonFast(in, &p); } } - TString TJsonPrettifier::Prettify(TStringBuf in) const { - TStringStream s; - if (Prettify(in, s)) - return s.Str(); - return TString(); + TString TJsonPrettifier::Prettify(TStringBuf in) const { + TStringStream s; + if (Prettify(in, s)) + return s.Str(); + return TString(); } } diff --git a/library/cpp/json/json_prettifier.h b/library/cpp/json/json_prettifier.h index 27d611b0b4..873942f11a 100644 --- a/library/cpp/json/json_prettifier.h +++ b/library/cpp/json/json_prettifier.h @@ -5,54 +5,54 @@ #include <util/generic/ylimits.h> namespace NJson { - struct TJsonPrettifier { - bool Unquote = false; - ui8 Padding = 4; - bool SingleQuotes = false; - bool Compactify = false; - bool Strict = false; - bool NewUnquote = false; // use new unquote, may break old tests - ui32 MaxPaddingLevel = Max<ui32>(); - - static TJsonPrettifier Prettifier(bool unquote = false, ui8 padding = 4, bool singlequotes = false) { - TJsonPrettifier p; - p.Unquote = unquote; - p.Padding = padding; - p.SingleQuotes = singlequotes; - return p; - } - - static TJsonPrettifier Compactifier(bool unquote = false, bool singlequote = false) { - TJsonPrettifier p; - p.Unquote = unquote; - p.Padding = 0; - p.Compactify = true; - p.SingleQuotes = singlequote; - return p; - } - - bool Prettify(TStringBuf in, IOutputStream& out) const; - - TString Prettify(TStringBuf in) const; - - static bool MayUnquoteNew(TStringBuf in); - static bool MayUnquoteOld(TStringBuf in); - }; - - inline TString PrettifyJson(TStringBuf in, bool unquote = false, ui8 padding = 4, bool sq = false) { - return TJsonPrettifier::Prettifier(unquote, padding, sq).Prettify(in); - } - - inline bool PrettifyJson(TStringBuf in, IOutputStream& out, bool unquote = false, ui8 padding = 4, bool sq = false) { - return TJsonPrettifier::Prettifier(unquote, padding, sq).Prettify(in, out); - } - - inline bool CompactifyJson(TStringBuf in, IOutputStream& out, bool unquote = false, bool sq = false) { - return TJsonPrettifier::Compactifier(unquote, sq).Prettify(in, out); - } - - inline TString CompactifyJson(TStringBuf in, bool unquote = false, bool sq = false) { - return TJsonPrettifier::Compactifier(unquote, sq).Prettify(in); - } + struct TJsonPrettifier { + bool Unquote = false; + ui8 Padding = 4; + bool SingleQuotes = false; + bool Compactify = false; + bool Strict = false; + bool NewUnquote = false; // use new unquote, may break old tests + ui32 MaxPaddingLevel = Max<ui32>(); + + static TJsonPrettifier Prettifier(bool unquote = false, ui8 padding = 4, bool singlequotes = false) { + TJsonPrettifier p; + p.Unquote = unquote; + p.Padding = padding; + p.SingleQuotes = singlequotes; + return p; + } + + static TJsonPrettifier Compactifier(bool unquote = false, bool singlequote = false) { + TJsonPrettifier p; + p.Unquote = unquote; + p.Padding = 0; + p.Compactify = true; + p.SingleQuotes = singlequote; + return p; + } + + bool Prettify(TStringBuf in, IOutputStream& out) const; + + TString Prettify(TStringBuf in) const; + + static bool MayUnquoteNew(TStringBuf in); + static bool MayUnquoteOld(TStringBuf in); + }; + + inline TString PrettifyJson(TStringBuf in, bool unquote = false, ui8 padding = 4, bool sq = false) { + return TJsonPrettifier::Prettifier(unquote, padding, sq).Prettify(in); + } + + inline bool PrettifyJson(TStringBuf in, IOutputStream& out, bool unquote = false, ui8 padding = 4, bool sq = false) { + return TJsonPrettifier::Prettifier(unquote, padding, sq).Prettify(in, out); + } + + inline bool CompactifyJson(TStringBuf in, IOutputStream& out, bool unquote = false, bool sq = false) { + return TJsonPrettifier::Compactifier(unquote, sq).Prettify(in, out); + } + + inline TString CompactifyJson(TStringBuf in, bool unquote = false, bool sq = false) { + return TJsonPrettifier::Compactifier(unquote, sq).Prettify(in); + } } diff --git a/library/cpp/json/json_reader.cpp b/library/cpp/json/json_reader.cpp index 072c8deafe..87ec7d2742 100644 --- a/library/cpp/json/json_reader.cpp +++ b/library/cpp/json/json_reader.cpp @@ -20,117 +20,117 @@ namespace NJson { } } - static const size_t DEFAULT_BUFFER_LEN = 65536; - - bool TParserCallbacks::OpenComplexValue(EJsonValueType type) { - TJsonValue* pvalue; - switch (CurrentState) { - case START: - Value.SetType(type); - ValuesStack.push_back(&Value); - break; - case IN_ARRAY: - pvalue = &ValuesStack.back()->AppendValue(type); - ValuesStack.push_back(pvalue); - break; - case AFTER_MAP_KEY: - pvalue = &ValuesStack.back()->InsertValue(Key, type); - ValuesStack.push_back(pvalue); - CurrentState = IN_MAP; - break; - default: - return false; - } - return true; - } - - bool TParserCallbacks::CloseComplexValue() { - if (ValuesStack.empty()) { + static const size_t DEFAULT_BUFFER_LEN = 65536; + + bool TParserCallbacks::OpenComplexValue(EJsonValueType type) { + TJsonValue* pvalue; + switch (CurrentState) { + case START: + Value.SetType(type); + ValuesStack.push_back(&Value); + break; + case IN_ARRAY: + pvalue = &ValuesStack.back()->AppendValue(type); + ValuesStack.push_back(pvalue); + break; + case AFTER_MAP_KEY: + pvalue = &ValuesStack.back()->InsertValue(Key, type); + ValuesStack.push_back(pvalue); + CurrentState = IN_MAP; + break; + default: + return false; + } + return true; + } + + bool TParserCallbacks::CloseComplexValue() { + if (ValuesStack.empty()) { return false; } - - ValuesStack.pop_back(); - if (!ValuesStack.empty()) { - switch (ValuesStack.back()->GetType()) { - case JSON_ARRAY: - CurrentState = IN_ARRAY; - break; - case JSON_MAP: - CurrentState = IN_MAP; - break; - default: - return false; - } - } else { - CurrentState = FINISH; - } - return true; + + ValuesStack.pop_back(); + if (!ValuesStack.empty()) { + switch (ValuesStack.back()->GetType()) { + case JSON_ARRAY: + CurrentState = IN_ARRAY; + break; + case JSON_MAP: + CurrentState = IN_MAP; + break; + default: + return false; + } + } else { + CurrentState = FINISH; + } + return true; } TParserCallbacks::TParserCallbacks(TJsonValue& value, bool throwOnError, bool notClosedBracketIsError) - : TJsonCallbacks(throwOnError) - , Value(value) + : TJsonCallbacks(throwOnError) + , Value(value) , NotClosedBracketIsError(notClosedBracketIsError) - , CurrentState(START) - { - } - - bool TParserCallbacks::OnNull() { - return SetValue(JSON_NULL); - } - - bool TParserCallbacks::OnBoolean(bool val) { - return SetValue(val); - } - - bool TParserCallbacks::OnInteger(long long val) { - return SetValue(val); - } - - bool TParserCallbacks::OnUInteger(unsigned long long val) { - return SetValue(val); - } - - bool TParserCallbacks::OnString(const TStringBuf& val) { - return SetValue(val); - } - - bool TParserCallbacks::OnDouble(double val) { - return SetValue(val); - } - - bool TParserCallbacks::OnOpenArray() { - bool res = OpenComplexValue(JSON_ARRAY); - if (res) - CurrentState = IN_ARRAY; - return res; - } - - bool TParserCallbacks::OnCloseArray() { - return CloseComplexValue(); - } - - bool TParserCallbacks::OnOpenMap() { - bool res = OpenComplexValue(JSON_MAP); - if (res) - CurrentState = IN_MAP; - return res; - } - - bool TParserCallbacks::OnCloseMap() { - return CloseComplexValue(); - } - - bool TParserCallbacks::OnMapKey(const TStringBuf& val) { - switch (CurrentState) { - case IN_MAP: - Key = val; - CurrentState = AFTER_MAP_KEY; - break; - default: - return false; - } - return true; + , CurrentState(START) + { + } + + bool TParserCallbacks::OnNull() { + return SetValue(JSON_NULL); + } + + bool TParserCallbacks::OnBoolean(bool val) { + return SetValue(val); + } + + bool TParserCallbacks::OnInteger(long long val) { + return SetValue(val); + } + + bool TParserCallbacks::OnUInteger(unsigned long long val) { + return SetValue(val); + } + + bool TParserCallbacks::OnString(const TStringBuf& val) { + return SetValue(val); + } + + bool TParserCallbacks::OnDouble(double val) { + return SetValue(val); + } + + bool TParserCallbacks::OnOpenArray() { + bool res = OpenComplexValue(JSON_ARRAY); + if (res) + CurrentState = IN_ARRAY; + return res; + } + + bool TParserCallbacks::OnCloseArray() { + return CloseComplexValue(); + } + + bool TParserCallbacks::OnOpenMap() { + bool res = OpenComplexValue(JSON_MAP); + if (res) + CurrentState = IN_MAP; + return res; + } + + bool TParserCallbacks::OnCloseMap() { + return CloseComplexValue(); + } + + bool TParserCallbacks::OnMapKey(const TStringBuf& val) { + switch (CurrentState) { + case IN_MAP: + Key = val; + CurrentState = AFTER_MAP_KEY; + break; + default: + return false; + } + return true; } bool TParserCallbacks::OnEnd() { @@ -140,182 +140,182 @@ namespace NJson { return true; } - TJsonReaderConfig::TJsonReaderConfig() - : BufferSize(DEFAULT_BUFFER_LEN) - { - } + TJsonReaderConfig::TJsonReaderConfig() + : BufferSize(DEFAULT_BUFFER_LEN) + { + } - void TJsonReaderConfig::SetBufferSize(size_t bufferSize) { - BufferSize = Max((size_t)1, Min(bufferSize, DEFAULT_BUFFER_LEN)); - } + void TJsonReaderConfig::SetBufferSize(size_t bufferSize) { + BufferSize = Max((size_t)1, Min(bufferSize, DEFAULT_BUFFER_LEN)); + } - size_t TJsonReaderConfig::GetBufferSize() const { - return BufferSize; - } + size_t TJsonReaderConfig::GetBufferSize() const { + return BufferSize; + } - namespace { - struct TJsonValueBuilder { + namespace { + struct TJsonValueBuilder { #ifdef NDEBUG - using TItem = TJsonValue*; + using TItem = TJsonValue*; - inline TJsonValue& Access(TItem& item) const { - return *item; - } + inline TJsonValue& Access(TItem& item) const { + return *item; + } #else - struct TItem { - TJsonValue* V; - size_t DuplicateKeyCount; - - TItem(TJsonValue* v) - : V(v) - , DuplicateKeyCount(0) - { - } - }; - - inline TJsonValue& Access(TItem& item) const { - return *item.V; + struct TItem { + TJsonValue* V; + size_t DuplicateKeyCount; + + TItem(TJsonValue* v) + : V(v) + , DuplicateKeyCount(0) + { + } + }; + + inline TJsonValue& Access(TItem& item) const { + return *item.V; } #endif - NJson::TJsonValue& V; - - TStack<TItem> S; - - TJsonValueBuilder(NJson::TJsonValue& v) - : V(v) - { - S.emplace(&V); - } - - template <class T> - void Set(const T& t) { - if (Access(S.top()).IsArray()) { - Access(S.top()).AppendValue(t); - } else { - Access(S.top()) = t; - S.pop(); - } - } - - bool Null() { - Set(NJson::JSON_NULL); - return true; - } - - bool Bool(bool b) { - Set(b); - return true; - } - - bool Int(int i) { - Set(i); - return true; - } - - template <class U> - bool ProcessUint(U u) { - if (Y_LIKELY(u <= static_cast<ui64>(Max<i64>()))) { - Set(i64(u)); - } else { - Set(u); - } - return true; - } - - bool Uint(unsigned u) { - return ProcessUint(u); - } - - bool Int64(i64 i) { - Set(i); - return true; - } - - bool Uint64(ui64 u) { - return ProcessUint(u); - } - - bool Double(double d) { - Set(d); - return true; - } - - bool RawNumber(const char* str, rapidjson::SizeType length, bool copy) { - Y_ASSERT(false && "this method should never be called"); - Y_UNUSED(str); - Y_UNUSED(length); - Y_UNUSED(copy); - return true; - } - - bool String(const char* str, rapidjson::SizeType length, bool copy) { - Y_ASSERT(copy); - Set(TStringBuf(str, length)); - return true; - } - - bool StartObject() { - if (Access(S.top()).IsArray()) { - S.emplace(&Access(S.top()).AppendValue(NJson::JSON_MAP)); - } else { - Access(S.top()).SetType(NJson::JSON_MAP); - } - return true; - } - - bool Key(const char* str, rapidjson::SizeType length, bool copy) { - Y_ASSERT(copy); - auto& value = Access(S.top())[TStringBuf(str, length)]; + NJson::TJsonValue& V; + + TStack<TItem> S; + + TJsonValueBuilder(NJson::TJsonValue& v) + : V(v) + { + S.emplace(&V); + } + + template <class T> + void Set(const T& t) { + if (Access(S.top()).IsArray()) { + Access(S.top()).AppendValue(t); + } else { + Access(S.top()) = t; + S.pop(); + } + } + + bool Null() { + Set(NJson::JSON_NULL); + return true; + } + + bool Bool(bool b) { + Set(b); + return true; + } + + bool Int(int i) { + Set(i); + return true; + } + + template <class U> + bool ProcessUint(U u) { + if (Y_LIKELY(u <= static_cast<ui64>(Max<i64>()))) { + Set(i64(u)); + } else { + Set(u); + } + return true; + } + + bool Uint(unsigned u) { + return ProcessUint(u); + } + + bool Int64(i64 i) { + Set(i); + return true; + } + + bool Uint64(ui64 u) { + return ProcessUint(u); + } + + bool Double(double d) { + Set(d); + return true; + } + + bool RawNumber(const char* str, rapidjson::SizeType length, bool copy) { + Y_ASSERT(false && "this method should never be called"); + Y_UNUSED(str); + Y_UNUSED(length); + Y_UNUSED(copy); + return true; + } + + bool String(const char* str, rapidjson::SizeType length, bool copy) { + Y_ASSERT(copy); + Set(TStringBuf(str, length)); + return true; + } + + bool StartObject() { + if (Access(S.top()).IsArray()) { + S.emplace(&Access(S.top()).AppendValue(NJson::JSON_MAP)); + } else { + Access(S.top()).SetType(NJson::JSON_MAP); + } + return true; + } + + bool Key(const char* str, rapidjson::SizeType length, bool copy) { + Y_ASSERT(copy); + auto& value = Access(S.top())[TStringBuf(str, length)]; if (Y_UNLIKELY(value.GetType() != JSON_UNDEFINED)) { #ifndef NDEBUG - ++S.top().DuplicateKeyCount; + ++S.top().DuplicateKeyCount; #endif - value.SetType(JSON_UNDEFINED); - } - S.emplace(&value); - return true; + value.SetType(JSON_UNDEFINED); + } + S.emplace(&value); + return true; } - inline int GetDuplicateKeyCount() const { + inline int GetDuplicateKeyCount() const { #ifdef NDEBUG - return 0; + return 0; #else - return S.top().DuplicateKeyCount; + return S.top().DuplicateKeyCount; #endif - } - - bool EndObject(rapidjson::SizeType memberCount) { - Y_ASSERT(memberCount == Access(S.top()).GetMap().size() + GetDuplicateKeyCount()); - S.pop(); - return true; - } - - bool StartArray() { - if (Access(S.top()).IsArray()) { - S.emplace(&Access(S.top()).AppendValue(NJson::JSON_ARRAY)); - } else { - Access(S.top()).SetType(NJson::JSON_ARRAY); - } - return true; - } - - bool EndArray(rapidjson::SizeType elementCount) { - Y_ASSERT(elementCount == Access(S.top()).GetArray().size()); - S.pop(); - return true; - } - }; - - template <class TRapidJsonCompliantInputStream, class THandler> - auto Read(const TJsonReaderConfig& config, - rapidjson::Reader& reader, - TRapidJsonCompliantInputStream& is, - THandler& handler) { + } + + bool EndObject(rapidjson::SizeType memberCount) { + Y_ASSERT(memberCount == Access(S.top()).GetMap().size() + GetDuplicateKeyCount()); + S.pop(); + return true; + } + + bool StartArray() { + if (Access(S.top()).IsArray()) { + S.emplace(&Access(S.top()).AppendValue(NJson::JSON_ARRAY)); + } else { + Access(S.top()).SetType(NJson::JSON_ARRAY); + } + return true; + } + + bool EndArray(rapidjson::SizeType elementCount) { + Y_ASSERT(elementCount == Access(S.top()).GetArray().size()); + S.pop(); + return true; + } + }; + + template <class TRapidJsonCompliantInputStream, class THandler> + auto Read(const TJsonReaderConfig& config, + rapidjson::Reader& reader, + TRapidJsonCompliantInputStream& is, + THandler& handler) { ui8 flags = ReaderConfigToRapidJsonFlags::NOCOMMENTS_VALID_NOESCAPE; - if (config.AllowComments) { + if (config.AllowComments) { flags |= ReaderConfigFlags::COMMENTS; } @@ -329,9 +329,9 @@ namespace NJson { switch (flags) { case ReaderConfigToRapidJsonFlags::COMMENTS_NOVALID_NOESCAPE: - return reader.Parse<rapidjson::kParseCommentsFlag>(is, handler); + return reader.Parse<rapidjson::kParseCommentsFlag>(is, handler); case ReaderConfigToRapidJsonFlags::COMMENTS_VALID_NOESCAPE: - return reader.Parse<rapidjson::kParseCommentsFlag | rapidjson::kParseValidateEncodingFlag>(is, handler); + return reader.Parse<rapidjson::kParseCommentsFlag | rapidjson::kParseValidateEncodingFlag>(is, handler); case ReaderConfigToRapidJsonFlags::COMMENTS_VALID_ESCAPE: return reader.Parse<rapidjson::kParseCommentsFlag | rapidjson::kParseValidateEncodingFlag | rapidjson::kParseEscapedApostropheFlag>(is, handler); case ReaderConfigToRapidJsonFlags::COMMENTS_NOVALID_ESCAPE: @@ -343,84 +343,84 @@ namespace NJson { case ReaderConfigToRapidJsonFlags::NOCOMMENTS_NOVALID_ESCAPE: return reader.Parse<rapidjson::kParseEscapedApostropheFlag>(is, handler); default: - return reader.Parse<rapidjson::kParseNoFlags>(is, handler); + return reader.Parse<rapidjson::kParseNoFlags>(is, handler); } } - template <class TRapidJsonCompliantInputStream, class THandler> - bool ReadJson(TRapidJsonCompliantInputStream& is, const TJsonReaderConfig* config, THandler& handler, bool throwOnError) { - rapidjson::Reader reader; - - auto result = Read(*config, reader, is, handler); - - if (result.IsError()) { - if (throwOnError) { + template <class TRapidJsonCompliantInputStream, class THandler> + bool ReadJson(TRapidJsonCompliantInputStream& is, const TJsonReaderConfig* config, THandler& handler, bool throwOnError) { + rapidjson::Reader reader; + + auto result = Read(*config, reader, is, handler); + + if (result.IsError()) { + if (throwOnError) { ythrow TJsonException() << PrintError(result); - } else { - return false; - } - } - + } else { + return false; + } + } + return true; } - template <class TRapidJsonCompliantInputStream> - bool ReadJsonTree(TRapidJsonCompliantInputStream& is, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError) { - out->SetType(NJson::JSON_NULL); - - TJsonValueBuilder handler(*out); - - return ReadJson(is, config, handler, throwOnError); - } - - template <class TData> - bool ReadJsonTreeImpl(TData* in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError) { - std::conditional_t<std::is_same<TData, TStringBuf>::value, TStringBufStreamWrapper, TInputStreamWrapper> is(*in); - return ReadJsonTree(is, config, out, throwOnError); - } - - template <class TData> - bool ReadJsonTreeImpl(TData* in, bool allowComments, TJsonValue* out, bool throwOnError) { - TJsonReaderConfig config; - config.AllowComments = allowComments; - return ReadJsonTreeImpl(in, &config, out, throwOnError); + template <class TRapidJsonCompliantInputStream> + bool ReadJsonTree(TRapidJsonCompliantInputStream& is, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError) { + out->SetType(NJson::JSON_NULL); + + TJsonValueBuilder handler(*out); + + return ReadJson(is, config, handler, throwOnError); } - template <class TData> - bool ReadJsonTreeImpl(TData* in, TJsonValue* out, bool throwOnError) { - return ReadJsonTreeImpl(in, false, out, throwOnError); + template <class TData> + bool ReadJsonTreeImpl(TData* in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError) { + std::conditional_t<std::is_same<TData, TStringBuf>::value, TStringBufStreamWrapper, TInputStreamWrapper> is(*in); + return ReadJsonTree(is, config, out, throwOnError); + } + + template <class TData> + bool ReadJsonTreeImpl(TData* in, bool allowComments, TJsonValue* out, bool throwOnError) { + TJsonReaderConfig config; + config.AllowComments = allowComments; + return ReadJsonTreeImpl(in, &config, out, throwOnError); + } + + template <class TData> + bool ReadJsonTreeImpl(TData* in, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(in, false, out, throwOnError); } - } //namespace + } //namespace - bool ReadJsonTree(TStringBuf in, TJsonValue* out, bool throwOnError) { - return ReadJsonTreeImpl(&in, out, throwOnError); + bool ReadJsonTree(TStringBuf in, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(&in, out, throwOnError); } - bool ReadJsonTree(TStringBuf in, bool allowComments, TJsonValue* out, bool throwOnError) { - return ReadJsonTreeImpl(&in, allowComments, out, throwOnError); - } + bool ReadJsonTree(TStringBuf in, bool allowComments, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(&in, allowComments, out, throwOnError); + } - bool ReadJsonTree(TStringBuf in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError) { - return ReadJsonTreeImpl(&in, config, out, throwOnError); + bool ReadJsonTree(TStringBuf in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(&in, config, out, throwOnError); } - bool ReadJsonTree(IInputStream* in, TJsonValue* out, bool throwOnError) { - return ReadJsonTreeImpl(in, out, throwOnError); + bool ReadJsonTree(IInputStream* in, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(in, out, throwOnError); } - bool ReadJsonTree(IInputStream* in, bool allowComments, TJsonValue* out, bool throwOnError) { - return ReadJsonTreeImpl(in, allowComments, out, throwOnError); + bool ReadJsonTree(IInputStream* in, bool allowComments, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(in, allowComments, out, throwOnError); } - bool ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError) { - return ReadJsonTreeImpl(in, config, out, throwOnError); + bool ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError) { + return ReadJsonTreeImpl(in, config, out, throwOnError); } bool ReadJsonFastTree(TStringBuf in, TJsonValue* out, bool throwOnError, bool notClosedBracketIsError) { TParserCallbacks cb(*out, throwOnError, notClosedBracketIsError); - return ReadJsonFast(in, &cb); - } + return ReadJsonFast(in, &cb); + } TJsonValue ReadJsonFastTree(TStringBuf in, bool notClosedBracketIsError) { TJsonValue value; @@ -429,99 +429,99 @@ namespace NJson { return value; } - namespace { - struct TJsonCallbacksWrapper { - TJsonCallbacks& Impl; - - TJsonCallbacksWrapper(TJsonCallbacks& impl) - : Impl(impl) - { - } - - bool Null() { - return Impl.OnNull(); - } - - bool Bool(bool b) { - return Impl.OnBoolean(b); - } - - template <class U> - bool ProcessUint(U u) { - if (Y_LIKELY(u <= ui64(Max<i64>()))) { - return Impl.OnInteger(i64(u)); - } else { - return Impl.OnUInteger(u); - } - } - - bool Int(int i) { - return Impl.OnInteger(i); - } - - bool Uint(unsigned u) { - return ProcessUint(u); - } - - bool Int64(i64 i) { - return Impl.OnInteger(i); - } - - bool Uint64(ui64 u) { - return ProcessUint(u); - } - - bool Double(double d) { - return Impl.OnDouble(d); - } - - bool RawNumber(const char* str, rapidjson::SizeType length, bool copy) { - Y_ASSERT(false && "this method should never be called"); - Y_UNUSED(str); - Y_UNUSED(length); - Y_UNUSED(copy); - return true; - } - - bool String(const char* str, rapidjson::SizeType length, bool copy) { - Y_ASSERT(copy); - return Impl.OnString(TStringBuf(str, length)); - } - - bool StartObject() { - return Impl.OnOpenMap(); - } - - bool Key(const char* str, rapidjson::SizeType length, bool copy) { - Y_ASSERT(copy); - return Impl.OnMapKey(TStringBuf(str, length)); - } - - bool EndObject(rapidjson::SizeType memberCount) { - Y_UNUSED(memberCount); - return Impl.OnCloseMap(); - } - - bool StartArray() { - return Impl.OnOpenArray(); - } - - bool EndArray(rapidjson::SizeType elementCount) { - Y_UNUSED(elementCount); - return Impl.OnCloseArray(); - } - }; - } - - bool ReadJson(IInputStream* in, TJsonCallbacks* cbs) { - return ReadJson(in, false, cbs); - } - - bool ReadJson(IInputStream* in, bool allowComments, TJsonCallbacks* cbs) { - TJsonReaderConfig config; - config.AllowComments = allowComments; - return ReadJson(in, &config, cbs); - } + namespace { + struct TJsonCallbacksWrapper { + TJsonCallbacks& Impl; + + TJsonCallbacksWrapper(TJsonCallbacks& impl) + : Impl(impl) + { + } + + bool Null() { + return Impl.OnNull(); + } + + bool Bool(bool b) { + return Impl.OnBoolean(b); + } + + template <class U> + bool ProcessUint(U u) { + if (Y_LIKELY(u <= ui64(Max<i64>()))) { + return Impl.OnInteger(i64(u)); + } else { + return Impl.OnUInteger(u); + } + } + + bool Int(int i) { + return Impl.OnInteger(i); + } + + bool Uint(unsigned u) { + return ProcessUint(u); + } + + bool Int64(i64 i) { + return Impl.OnInteger(i); + } + + bool Uint64(ui64 u) { + return ProcessUint(u); + } + + bool Double(double d) { + return Impl.OnDouble(d); + } + + bool RawNumber(const char* str, rapidjson::SizeType length, bool copy) { + Y_ASSERT(false && "this method should never be called"); + Y_UNUSED(str); + Y_UNUSED(length); + Y_UNUSED(copy); + return true; + } + + bool String(const char* str, rapidjson::SizeType length, bool copy) { + Y_ASSERT(copy); + return Impl.OnString(TStringBuf(str, length)); + } + + bool StartObject() { + return Impl.OnOpenMap(); + } + + bool Key(const char* str, rapidjson::SizeType length, bool copy) { + Y_ASSERT(copy); + return Impl.OnMapKey(TStringBuf(str, length)); + } + + bool EndObject(rapidjson::SizeType memberCount) { + Y_UNUSED(memberCount); + return Impl.OnCloseMap(); + } + + bool StartArray() { + return Impl.OnOpenArray(); + } + + bool EndArray(rapidjson::SizeType elementCount) { + Y_UNUSED(elementCount); + return Impl.OnCloseArray(); + } + }; + } + + bool ReadJson(IInputStream* in, TJsonCallbacks* cbs) { + return ReadJson(in, false, cbs); + } + + bool ReadJson(IInputStream* in, bool allowComments, TJsonCallbacks* cbs) { + TJsonReaderConfig config; + config.AllowComments = allowComments; + return ReadJson(in, &config, cbs); + } bool ReadJson(IInputStream* in, bool allowComments, bool allowEscapedApostrophe, TJsonCallbacks* cbs) { TJsonReaderConfig config; @@ -530,38 +530,38 @@ namespace NJson { return ReadJson(in, &config, cbs); } - bool ReadJson(IInputStream* in, const TJsonReaderConfig* config, TJsonCallbacks* cbs) { - TJsonCallbacksWrapper wrapper(*cbs); - TInputStreamWrapper is(*in); + bool ReadJson(IInputStream* in, const TJsonReaderConfig* config, TJsonCallbacks* cbs) { + TJsonCallbacksWrapper wrapper(*cbs); + TInputStreamWrapper is(*in); - rapidjson::Reader reader; - auto result = Read(*config, reader, is, wrapper); + rapidjson::Reader reader; + auto result = Read(*config, reader, is, wrapper); - if (result.IsError()) { + if (result.IsError()) { cbs->OnError(result.Offset(), PrintError(result)); - return false; + return false; } - return cbs->OnEnd(); - } - - TJsonValue ReadJsonTree(IInputStream* in, bool throwOnError) { - TJsonValue out; - ReadJsonTree(in, &out, throwOnError); - return out; - } - - TJsonValue ReadJsonTree(IInputStream* in, bool allowComments, bool throwOnError) { - TJsonValue out; - ReadJsonTree(in, allowComments, &out, throwOnError); - return out; - } - - TJsonValue ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, bool throwOnError) { - TJsonValue out; - ReadJsonTree(in, config, &out, throwOnError); - return out; + return cbs->OnEnd(); + } + + TJsonValue ReadJsonTree(IInputStream* in, bool throwOnError) { + TJsonValue out; + ReadJsonTree(in, &out, throwOnError); + return out; + } + + TJsonValue ReadJsonTree(IInputStream* in, bool allowComments, bool throwOnError) { + TJsonValue out; + ReadJsonTree(in, allowComments, &out, throwOnError); + return out; + } + + TJsonValue ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, bool throwOnError) { + TJsonValue out; + ReadJsonTree(in, config, &out, throwOnError); + return out; } } diff --git a/library/cpp/json/json_reader.h b/library/cpp/json/json_reader.h index b673788330..674b3f6abf 100644 --- a/library/cpp/json/json_reader.h +++ b/library/cpp/json/json_reader.h @@ -4,7 +4,7 @@ #include <library/cpp/json/common/defs.h> #include <library/cpp/json/fast_sax/parser.h> - + #include <util/generic/yexception.h> #include <util/stream/input.h> @@ -12,37 +12,37 @@ #include <util/stream/mem.h> namespace NJson { - struct TJsonReaderConfig { - TJsonReaderConfig(); + struct TJsonReaderConfig { + TJsonReaderConfig(); - // js-style comments (both // and /**/) - bool AllowComments = false; - bool DontValidateUtf8 = false; + // js-style comments (both // and /**/) + bool AllowComments = false; + bool DontValidateUtf8 = false; bool AllowEscapedApostrophe = false; - void SetBufferSize(size_t bufferSize); - size_t GetBufferSize() const; + void SetBufferSize(size_t bufferSize); + size_t GetBufferSize() const; - private: - size_t BufferSize; - }; + private: + size_t BufferSize; + }; - bool ReadJsonTree(TStringBuf in, TJsonValue* out, bool throwOnError = false); - bool ReadJsonTree(TStringBuf in, bool allowComments, TJsonValue* out, bool throwOnError = false); - bool ReadJsonTree(TStringBuf in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError = false); + bool ReadJsonTree(TStringBuf in, TJsonValue* out, bool throwOnError = false); + bool ReadJsonTree(TStringBuf in, bool allowComments, TJsonValue* out, bool throwOnError = false); + bool ReadJsonTree(TStringBuf in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError = false); - bool ReadJsonTree(IInputStream* in, TJsonValue* out, bool throwOnError = false); - bool ReadJsonTree(IInputStream* in, bool allowComments, TJsonValue* out, bool throwOnError = false); - bool ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError = false); + bool ReadJsonTree(IInputStream* in, TJsonValue* out, bool throwOnError = false); + bool ReadJsonTree(IInputStream* in, bool allowComments, TJsonValue* out, bool throwOnError = false); + bool ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, TJsonValue* out, bool throwOnError = false); - TJsonValue ReadJsonTree(IInputStream* in, bool throwOnError = false); - TJsonValue ReadJsonTree(IInputStream* in, bool allowComments, bool throwOnError); - TJsonValue ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, bool throwOnError = false); + TJsonValue ReadJsonTree(IInputStream* in, bool throwOnError = false); + TJsonValue ReadJsonTree(IInputStream* in, bool allowComments, bool throwOnError); + TJsonValue ReadJsonTree(IInputStream* in, const TJsonReaderConfig* config, bool throwOnError = false); - bool ReadJson(IInputStream* in, TJsonCallbacks* callbacks); - bool ReadJson(IInputStream* in, bool allowComments, TJsonCallbacks* callbacks); + bool ReadJson(IInputStream* in, TJsonCallbacks* callbacks); + bool ReadJson(IInputStream* in, bool allowComments, TJsonCallbacks* callbacks); bool ReadJson(IInputStream* in, bool allowComments, bool allowEscapedApostrophe, TJsonCallbacks* callbacks); - bool ReadJson(IInputStream* in, const TJsonReaderConfig* config, TJsonCallbacks* callbacks); + bool ReadJson(IInputStream* in, const TJsonReaderConfig* config, TJsonCallbacks* callbacks); enum ReaderConfigFlags { COMMENTS = 0b100, @@ -60,79 +60,79 @@ namespace NJson { NOCOMMENTS_NOVALID_ESCAPE = 0b001, }; - inline bool ValidateJson(IInputStream* in, const TJsonReaderConfig* config, bool throwOnError = false) { - TJsonCallbacks c(throwOnError); - return ReadJson(in, config, &c); - } + inline bool ValidateJson(IInputStream* in, const TJsonReaderConfig* config, bool throwOnError = false) { + TJsonCallbacks c(throwOnError); + return ReadJson(in, config, &c); + } - inline bool ValidateJson(TStringBuf in, const TJsonReaderConfig& config = TJsonReaderConfig(), bool throwOnError = false) { + inline bool ValidateJson(TStringBuf in, const TJsonReaderConfig& config = TJsonReaderConfig(), bool throwOnError = false) { TMemoryInput min(in.data(), in.size()); - return ValidateJson(&min, &config, throwOnError); - } + return ValidateJson(&min, &config, throwOnError); + } - inline bool ValidateJsonThrow(IInputStream* in, const TJsonReaderConfig* config) { - return ValidateJson(in, config, true); - } + inline bool ValidateJsonThrow(IInputStream* in, const TJsonReaderConfig* config) { + return ValidateJson(in, config, true); + } - inline bool ValidateJsonThrow(TStringBuf in, const TJsonReaderConfig& config = TJsonReaderConfig()) { - return ValidateJson(in, config, true); - } + inline bool ValidateJsonThrow(TStringBuf in, const TJsonReaderConfig& config = TJsonReaderConfig()) { + return ValidateJson(in, config, true); + } - class TParserCallbacks: public TJsonCallbacks { - public: + class TParserCallbacks: public TJsonCallbacks { + public: TParserCallbacks(TJsonValue& value, bool throwOnError = false, bool notClosedBracketIsError = false); - bool OnNull() override; - bool OnBoolean(bool val) override; - bool OnInteger(long long val) override; - bool OnUInteger(unsigned long long val) override; - bool OnString(const TStringBuf& val) override; - bool OnDouble(double val) override; - bool OnOpenArray() override; - bool OnCloseArray() override; - bool OnOpenMap() override; - bool OnCloseMap() override; - bool OnMapKey(const TStringBuf& val) override; + bool OnNull() override; + bool OnBoolean(bool val) override; + bool OnInteger(long long val) override; + bool OnUInteger(unsigned long long val) override; + bool OnString(const TStringBuf& val) override; + bool OnDouble(double val) override; + bool OnOpenArray() override; + bool OnCloseArray() override; + bool OnOpenMap() override; + bool OnCloseMap() override; + bool OnMapKey(const TStringBuf& val) override; bool OnEnd() override; - protected: - TJsonValue& Value; - TString Key; - TVector<TJsonValue*> ValuesStack; + protected: + TJsonValue& Value; + TString Key; + TVector<TJsonValue*> ValuesStack; bool NotClosedBracketIsError; - enum { - START, - AFTER_MAP_KEY, - IN_MAP, - IN_ARRAY, - FINISH - } CurrentState; - - template <class T> - bool SetValue(const T& value) { - switch (CurrentState) { - case START: - Value.SetValue(value); - break; - case AFTER_MAP_KEY: - ValuesStack.back()->InsertValue(Key, value); - CurrentState = IN_MAP; - break; - case IN_ARRAY: - ValuesStack.back()->AppendValue(value); - break; - case IN_MAP: - case FINISH: - return false; - default: - ythrow yexception() << "TParserCallbacks::SetValue invalid enum"; - } - return true; + enum { + START, + AFTER_MAP_KEY, + IN_MAP, + IN_ARRAY, + FINISH + } CurrentState; + + template <class T> + bool SetValue(const T& value) { + switch (CurrentState) { + case START: + Value.SetValue(value); + break; + case AFTER_MAP_KEY: + ValuesStack.back()->InsertValue(Key, value); + CurrentState = IN_MAP; + break; + case IN_ARRAY: + ValuesStack.back()->AppendValue(value); + break; + case IN_MAP: + case FINISH: + return false; + default: + ythrow yexception() << "TParserCallbacks::SetValue invalid enum"; + } + return true; } - bool OpenComplexValue(EJsonValueType type); - bool CloseComplexValue(); - }; + bool OpenComplexValue(EJsonValueType type); + bool CloseComplexValue(); + }; //// relaxed json, used in library/cpp/scheme bool ReadJsonFastTree(TStringBuf in, TJsonValue* out, bool throwOnError = false, bool notClosedBracketIsError = false); diff --git a/library/cpp/json/json_writer.cpp b/library/cpp/json/json_writer.cpp index 3d058bae36..140e1427d3 100644 --- a/library/cpp/json/json_writer.cpp +++ b/library/cpp/json/json_writer.cpp @@ -6,144 +6,144 @@ #include <util/system/yassert.h> namespace NJson { - TJsonWriter::TJsonWriter(IOutputStream* out, bool formatOutput, bool sortkeys, bool validateUtf8) - : Out(out) - , Buf(NJsonWriter::HEM_UNSAFE) + TJsonWriter::TJsonWriter(IOutputStream* out, bool formatOutput, bool sortkeys, bool validateUtf8) + : Out(out) + , Buf(NJsonWriter::HEM_UNSAFE) , DoubleNDigits(TJsonWriterConfig::DefaultDoubleNDigits) , FloatNDigits(TJsonWriterConfig::DefaultFloatNDigits) , FloatToStringMode(TJsonWriterConfig::DefaultFloatToStringMode) - , SortKeys(sortkeys) - , ValidateUtf8(validateUtf8) - , DontEscapeStrings(false) - , DontFlushInDestructor(false) - { - Buf.SetIndentSpaces(formatOutput ? 2 : 0); - } - - TJsonWriter::TJsonWriter(IOutputStream* out, const TJsonWriterConfig& config, bool DFID) - : Out(config.Unbuffered ? nullptr : out) - , Buf(NJsonWriter::HEM_UNSAFE, config.Unbuffered ? out : nullptr) + , SortKeys(sortkeys) + , ValidateUtf8(validateUtf8) + , DontEscapeStrings(false) + , DontFlushInDestructor(false) + { + Buf.SetIndentSpaces(formatOutput ? 2 : 0); + } + + TJsonWriter::TJsonWriter(IOutputStream* out, const TJsonWriterConfig& config, bool DFID) + : Out(config.Unbuffered ? nullptr : out) + , Buf(NJsonWriter::HEM_UNSAFE, config.Unbuffered ? out : nullptr) , DoubleNDigits(config.DoubleNDigits) , FloatNDigits(config.FloatNDigits) , FloatToStringMode(config.FloatToStringMode) - , SortKeys(config.SortKeys) - , ValidateUtf8(config.ValidateUtf8) - , DontEscapeStrings(config.DontEscapeStrings) - , DontFlushInDestructor(DFID) - { - Buf.SetIndentSpaces(config.FormatOutput ? 2 : 0); - Buf.SetWriteNanAsString(config.WriteNanAsString); - } - - TJsonWriter::~TJsonWriter() { - // if we write to socket it's possible to get exception here - // don't use exceptions in destructors - if (!DontFlushInDestructor) { - try { - Flush(); - } catch (...) { - } - } - } - - void TJsonWriter::Flush() { - if (Out) { - Buf.FlushTo(Out); + , SortKeys(config.SortKeys) + , ValidateUtf8(config.ValidateUtf8) + , DontEscapeStrings(config.DontEscapeStrings) + , DontFlushInDestructor(DFID) + { + Buf.SetIndentSpaces(config.FormatOutput ? 2 : 0); + Buf.SetWriteNanAsString(config.WriteNanAsString); + } + + TJsonWriter::~TJsonWriter() { + // if we write to socket it's possible to get exception here + // don't use exceptions in destructors + if (!DontFlushInDestructor) { + try { + Flush(); + } catch (...) { + } + } + } + + void TJsonWriter::Flush() { + if (Out) { + Buf.FlushTo(Out); } - } + } - void TJsonWriter::OpenMap() { - Buf.BeginObject(); - } + void TJsonWriter::OpenMap() { + Buf.BeginObject(); + } - void TJsonWriter::CloseMap() { - Buf.EndObject(); - } + void TJsonWriter::CloseMap() { + Buf.EndObject(); + } - void TJsonWriter::OpenArray() { - Buf.BeginList(); - } + void TJsonWriter::OpenArray() { + Buf.BeginList(); + } - void TJsonWriter::CloseArray() { - Buf.EndList(); - } + void TJsonWriter::CloseArray() { + Buf.EndList(); + } - void TJsonWriter::Write(const TStringBuf& value) { - if (ValidateUtf8 && !IsUtf(value)) - throw yexception() << "JSON writer: invalid UTF-8"; - if (Buf.KeyExpected()) { - Buf.WriteKey(value); + void TJsonWriter::Write(const TStringBuf& value) { + if (ValidateUtf8 && !IsUtf(value)) + throw yexception() << "JSON writer: invalid UTF-8"; + if (Buf.KeyExpected()) { + Buf.WriteKey(value); } else { - if (DontEscapeStrings) { - Buf.UnsafeWriteValue(TString("\"") + value + '"'); - } else { - Buf.WriteString(value); - } + if (DontEscapeStrings) { + Buf.UnsafeWriteValue(TString("\"") + value + '"'); + } else { + Buf.WriteString(value); + } } } - void TJsonWriter::WriteNull() { - Buf.WriteNull(); - } + void TJsonWriter::WriteNull() { + Buf.WriteNull(); + } - void TJsonWriter::Write(float value) { + void TJsonWriter::Write(float value) { Buf.WriteFloat(value, FloatToStringMode, FloatNDigits); - } + } - void TJsonWriter::Write(double value) { + void TJsonWriter::Write(double value) { Buf.WriteDouble(value, FloatToStringMode, DoubleNDigits); - } + } - void TJsonWriter::Write(long long value) { - Buf.WriteLongLong(value); - } + void TJsonWriter::Write(long long value) { + Buf.WriteLongLong(value); + } - void TJsonWriter::Write(unsigned long long value) { - Buf.WriteULongLong(value); - } + void TJsonWriter::Write(unsigned long long value) { + Buf.WriteULongLong(value); + } - void TJsonWriter::Write(bool value) { - Buf.WriteBool(value); - } + void TJsonWriter::Write(bool value) { + Buf.WriteBool(value); + } - namespace { - struct TLessStrPtr { - bool operator()(const TString* a, const TString* b) const { - return *a < *b; - } - }; - } + namespace { + struct TLessStrPtr { + bool operator()(const TString* a, const TString* b) const { + return *a < *b; + } + }; + } - void TJsonWriter::Write(const TJsonValue* v) { + void TJsonWriter::Write(const TJsonValue* v) { Buf.WriteJsonValue(v, SortKeys, FloatToStringMode, DoubleNDigits); - } + } void TJsonWriter::Write(const TJsonValue& v) { Buf.WriteJsonValue(&v, SortKeys, FloatToStringMode, DoubleNDigits); } - TString WriteJson(const TJsonValue* value, bool formatOutput, bool sortkeys, bool validateUtf8) { - TStringStream ss; - WriteJson(&ss, value, formatOutput, sortkeys, validateUtf8); - return ss.Str(); - } - - TString WriteJson(const TJsonValue& value, bool formatOutput, bool sortkeys, bool validateUtf8) { - TStringStream ss; - WriteJson(&ss, &value, formatOutput, sortkeys, validateUtf8); - return ss.Str(); - } - - void WriteJson(IOutputStream* out, const TJsonValue* val, bool formatOutput, bool sortkeys, bool validateUtf8) { - TJsonWriter w(out, formatOutput, sortkeys, validateUtf8); - w.Write(val); - w.Flush(); - } - - void WriteJson(IOutputStream* out, const TJsonValue* val, const TJsonWriterConfig& config) { - TJsonWriter w(out, config, true); - w.Write(val); - w.Flush(); - } + TString WriteJson(const TJsonValue* value, bool formatOutput, bool sortkeys, bool validateUtf8) { + TStringStream ss; + WriteJson(&ss, value, formatOutput, sortkeys, validateUtf8); + return ss.Str(); + } + + TString WriteJson(const TJsonValue& value, bool formatOutput, bool sortkeys, bool validateUtf8) { + TStringStream ss; + WriteJson(&ss, &value, formatOutput, sortkeys, validateUtf8); + return ss.Str(); + } + + void WriteJson(IOutputStream* out, const TJsonValue* val, bool formatOutput, bool sortkeys, bool validateUtf8) { + TJsonWriter w(out, formatOutput, sortkeys, validateUtf8); + w.Write(val); + w.Flush(); + } + + void WriteJson(IOutputStream* out, const TJsonValue* val, const TJsonWriterConfig& config) { + TJsonWriter w(out, config, true); + w.Write(val); + w.Flush(); + } } diff --git a/library/cpp/json/json_writer.h b/library/cpp/json/json_writer.h index c7f5c9499a..b56ba5e8a7 100644 --- a/library/cpp/json/json_writer.h +++ b/library/cpp/json/json_writer.h @@ -12,148 +12,148 @@ #include <util/generic/strbuf.h> namespace NJson { - struct TJsonWriterConfig { + struct TJsonWriterConfig { constexpr static ui32 DefaultDoubleNDigits = 10; constexpr static ui32 DefaultFloatNDigits = 6; constexpr static EFloatToStringMode DefaultFloatToStringMode = PREC_NDIGITS; - inline TJsonWriterConfig& SetUnbuffered(bool v) noexcept { - Unbuffered = v; - - return *this; - } - - inline TJsonWriterConfig& SetValidateUtf8(bool v) noexcept { - ValidateUtf8 = v; - - return *this; - } - - inline TJsonWriterConfig& SetFormatOutput(bool v) noexcept { - FormatOutput = v; - - return *this; - } - + inline TJsonWriterConfig& SetUnbuffered(bool v) noexcept { + Unbuffered = v; + + return *this; + } + + inline TJsonWriterConfig& SetValidateUtf8(bool v) noexcept { + ValidateUtf8 = v; + + return *this; + } + + inline TJsonWriterConfig& SetFormatOutput(bool v) noexcept { + FormatOutput = v; + + return *this; + } + ui32 DoubleNDigits = DefaultDoubleNDigits; ui32 FloatNDigits = DefaultFloatNDigits; EFloatToStringMode FloatToStringMode = DefaultFloatToStringMode; - bool FormatOutput = false; - bool SortKeys = false; - bool ValidateUtf8 = true; - bool DontEscapeStrings = false; - bool Unbuffered = false; - bool WriteNanAsString = false; // NaN and Inf are not valid json values, so if WriteNanAsString is set, writer would write string intead of throwing exception (default case) - }; - - class TJsonWriter { - IOutputStream* Out; - NJsonWriter::TBuf Buf; + bool FormatOutput = false; + bool SortKeys = false; + bool ValidateUtf8 = true; + bool DontEscapeStrings = false; + bool Unbuffered = false; + bool WriteNanAsString = false; // NaN and Inf are not valid json values, so if WriteNanAsString is set, writer would write string intead of throwing exception (default case) + }; + + class TJsonWriter { + IOutputStream* Out; + NJsonWriter::TBuf Buf; const ui32 DoubleNDigits; const ui32 FloatNDigits; const EFloatToStringMode FloatToStringMode; - const bool SortKeys; - const bool ValidateUtf8; - const bool DontEscapeStrings; - const bool DontFlushInDestructor; - - public: - TJsonWriter(IOutputStream* out, bool formatOutput, bool sortkeys = false, bool validateUtf8 = true); - TJsonWriter(IOutputStream* out, const TJsonWriterConfig& config, bool DontFlushInDestructor = false); - ~TJsonWriter(); - - void Flush(); - - void OpenMap(); - void OpenMap(const TStringBuf& key) { - Buf.WriteKey(key); - OpenMap(); - } - void CloseMap(); - - void OpenArray(); - void OpenArray(const TStringBuf& key) { - Buf.WriteKey(key); - OpenArray(); - } - void CloseArray(); - - void WriteNull(); - - void Write(const TStringBuf& value); - void Write(float value); - void Write(double value); - void Write(bool value); - void Write(const TJsonValue* value); + const bool SortKeys; + const bool ValidateUtf8; + const bool DontEscapeStrings; + const bool DontFlushInDestructor; + + public: + TJsonWriter(IOutputStream* out, bool formatOutput, bool sortkeys = false, bool validateUtf8 = true); + TJsonWriter(IOutputStream* out, const TJsonWriterConfig& config, bool DontFlushInDestructor = false); + ~TJsonWriter(); + + void Flush(); + + void OpenMap(); + void OpenMap(const TStringBuf& key) { + Buf.WriteKey(key); + OpenMap(); + } + void CloseMap(); + + void OpenArray(); + void OpenArray(const TStringBuf& key) { + Buf.WriteKey(key); + OpenArray(); + } + void CloseArray(); + + void WriteNull(); + + void Write(const TStringBuf& value); + void Write(float value); + void Write(double value); + void Write(bool value); + void Write(const TJsonValue* value); void Write(const TJsonValue& value); - // must use all variations of integer types since long - // and long long are different types but with same size - void Write(long long value); - void Write(unsigned long long value); - void Write(long value) { - Write((long long)value); - } - void Write(unsigned long value) { - Write((unsigned long long)value); - } - void Write(int value) { - Write((long long)value); - } - void Write(unsigned int value) { - Write((unsigned long long)value); - } - void Write(short value) { - Write((long long)value); - } - void Write(unsigned short value) { - Write((unsigned long long)value); - } - - void Write(const unsigned char* value) { - Write((const char*)value); - } - void Write(const char* value) { - Write(TStringBuf(value)); - } - void Write(const TString& value) { - Write(TStringBuf(value)); - } - void Write(const std::string& value) { - Write(TStringBuf(value)); - } - - // write raw json without checks - void UnsafeWrite(const TStringBuf& value) { - Buf.UnsafeWriteValue(value); - } - - template <typename T> - void Write(const TStringBuf& key, const T& value) { - Buf.WriteKey(key); - Write(value); - } - - // write raw json without checks - void UnsafeWrite(const TStringBuf& key, const TStringBuf& value) { - Buf.WriteKey(key); - UnsafeWrite(value); - } - - void WriteNull(const TStringBuf& key) { - Buf.WriteKey(key); - WriteNull(); - } - - template <typename T> - void WriteOptional(const TStringBuf& key, const TMaybe<T>& value) { - if (value) { - Write(key, *value); - } - } - - void WriteOptional(const TStringBuf&, const TNothing&) { - // nothing to do + // must use all variations of integer types since long + // and long long are different types but with same size + void Write(long long value); + void Write(unsigned long long value); + void Write(long value) { + Write((long long)value); + } + void Write(unsigned long value) { + Write((unsigned long long)value); + } + void Write(int value) { + Write((long long)value); + } + void Write(unsigned int value) { + Write((unsigned long long)value); + } + void Write(short value) { + Write((long long)value); + } + void Write(unsigned short value) { + Write((unsigned long long)value); + } + + void Write(const unsigned char* value) { + Write((const char*)value); + } + void Write(const char* value) { + Write(TStringBuf(value)); + } + void Write(const TString& value) { + Write(TStringBuf(value)); + } + void Write(const std::string& value) { + Write(TStringBuf(value)); + } + + // write raw json without checks + void UnsafeWrite(const TStringBuf& value) { + Buf.UnsafeWriteValue(value); + } + + template <typename T> + void Write(const TStringBuf& key, const T& value) { + Buf.WriteKey(key); + Write(value); + } + + // write raw json without checks + void UnsafeWrite(const TStringBuf& key, const TStringBuf& value) { + Buf.WriteKey(key); + UnsafeWrite(value); + } + + void WriteNull(const TStringBuf& key) { + Buf.WriteKey(key); + WriteNull(); + } + + template <typename T> + void WriteOptional(const TStringBuf& key, const TMaybe<T>& value) { + if (value) { + Write(key, *value); + } + } + + void WriteOptional(const TStringBuf&, const TNothing&) { + // nothing to do } void WriteKey(const TStringBuf key) { @@ -172,25 +172,25 @@ namespace NJson { WriteKey(TStringBuf{key}); } - void WriteKey(const std::string& key) { - WriteKey(TStringBuf{key}); - } - - NJsonWriter::TBufState State() const { - return Buf.State(); - } + void WriteKey(const std::string& key) { + WriteKey(TStringBuf{key}); + } + + NJsonWriter::TBufState State() const { + return Buf.State(); + } - void Reset(const NJsonWriter::TBufState& from) { - return Buf.Reset(from); - } + void Reset(const NJsonWriter::TBufState& from) { + return Buf.Reset(from); + } - void Reset(NJsonWriter::TBufState&& from) { - return Buf.Reset(std::move(from)); - } - }; + void Reset(NJsonWriter::TBufState&& from) { + return Buf.Reset(std::move(from)); + } + }; - void WriteJson(IOutputStream*, const TJsonValue*, bool formatOutput = false, bool sortkeys = false, bool validateUtf8 = true); - TString WriteJson(const TJsonValue*, bool formatOutput = true, bool sortkeys = false, bool validateUtf8 = false); - TString WriteJson(const TJsonValue&, bool formatOutput = true, bool sortkeys = false, bool validateUtf8 = false); - void WriteJson(IOutputStream*, const TJsonValue*, const TJsonWriterConfig& config); + void WriteJson(IOutputStream*, const TJsonValue*, bool formatOutput = false, bool sortkeys = false, bool validateUtf8 = true); + TString WriteJson(const TJsonValue*, bool formatOutput = true, bool sortkeys = false, bool validateUtf8 = false); + TString WriteJson(const TJsonValue&, bool formatOutput = true, bool sortkeys = false, bool validateUtf8 = false); + void WriteJson(IOutputStream*, const TJsonValue*, const TJsonWriterConfig& config); } diff --git a/library/cpp/json/rapidjson_helpers.h b/library/cpp/json/rapidjson_helpers.h index aeb96ff670..6adad9b32f 100644 --- a/library/cpp/json/rapidjson_helpers.h +++ b/library/cpp/json/rapidjson_helpers.h @@ -26,7 +26,7 @@ namespace NJson { } }; - struct TInputStreamWrapper : TReadOnlyStreamBase { + struct TInputStreamWrapper : TReadOnlyStreamBase { Ch Peek() const { if (!Eof) { if (Pos >= Sz) { @@ -77,7 +77,7 @@ namespace NJson { size_t Count; }; - struct TStringBufStreamWrapper : TReadOnlyStreamBase { + struct TStringBufStreamWrapper : TReadOnlyStreamBase { Ch Peek() const { return Pos < Data.size() ? Data[Pos] : 0; } diff --git a/library/cpp/json/ut/json_prettifier_ut.cpp b/library/cpp/json/ut/json_prettifier_ut.cpp index ae5f8dd81a..e2b51be062 100644 --- a/library/cpp/json/ut/json_prettifier_ut.cpp +++ b/library/cpp/json/ut/json_prettifier_ut.cpp @@ -36,78 +36,78 @@ Y_UNIT_TEST_SUITE(JsonPrettifier) { Y_UNIT_TEST(PrettifyJsonLong) { UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("[{k:v},{a:b}]", false, 2, true), - "[\n" - " {\n" - " 'k' : 'v'\n" - " },\n" - " {\n" - " 'a' : 'b'\n" - " }\n" - "]"); + "[\n" + " {\n" + " 'k' : 'v'\n" + " },\n" + " {\n" + " 'a' : 'b'\n" + " }\n" + "]"); UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{k:v,a:b,x:[1,2,3]}", false, 2, true), - "{\n" - " 'k' : 'v',\n" - " 'a' : 'b',\n" - " 'x' : [\n" - " 1,\n" - " 2,\n" - " 3\n" - " ]\n" - "}"); + "{\n" + " 'k' : 'v',\n" + " 'a' : 'b',\n" + " 'x' : [\n" + " 1,\n" + " 2,\n" + " 3\n" + " ]\n" + "}"); UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{k:v,a:b,x:[1,{f:b},3],m:n}", false, 2, true), - "{\n" - " 'k' : 'v',\n" - " 'a' : 'b',\n" - " 'x' : [\n" - " 1,\n" - " {\n" - " 'f' : 'b'\n" - " },\n" - " 3\n" - " ],\n" - " 'm' : 'n'\n" - "}"); + "{\n" + " 'k' : 'v',\n" + " 'a' : 'b',\n" + " 'x' : [\n" + " 1,\n" + " {\n" + " 'f' : 'b'\n" + " },\n" + " 3\n" + " ],\n" + " 'm' : 'n'\n" + "}"); NJson::TJsonPrettifier prettifierMaxLevel1 = NJson::TJsonPrettifier::Prettifier(false, 2, true); prettifierMaxLevel1.MaxPaddingLevel = 1; UNIT_ASSERT_STRINGS_EQUAL(prettifierMaxLevel1.Prettify("{k:v,a:b,x:[1,{f:b},3],m:n}"), - "{\n" - " 'k' : 'v',\n" - " 'a' : 'b',\n" - " 'x' : [ 1, { 'f' : 'b' }, 3 ],\n" - " 'm' : 'n'\n" - "}"); + "{\n" + " 'k' : 'v',\n" + " 'a' : 'b',\n" + " 'x' : [ 1, { 'f' : 'b' }, 3 ],\n" + " 'm' : 'n'\n" + "}"); UNIT_ASSERT_STRINGS_EQUAL(NJson::PrettifyJson("{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}", true, 2), - "{\n" - " g : {\n" - " x : {\n" - " a : {\n" - " b : c,\n" - " e : f\n" - " },\n" - " q : {\n" - " x : y\n" - " }\n" - " },\n" - " y : fff\n" - " }\n" - "}"); + "{\n" + " g : {\n" + " x : {\n" + " a : {\n" + " b : c,\n" + " e : f\n" + " },\n" + " q : {\n" + " x : y\n" + " }\n" + " },\n" + " y : fff\n" + " }\n" + "}"); NJson::TJsonPrettifier prettifierMaxLevel3 = NJson::TJsonPrettifier::Prettifier(true, 2); prettifierMaxLevel3.MaxPaddingLevel = 3; UNIT_ASSERT_STRINGS_EQUAL(prettifierMaxLevel3.Prettify("{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}"), - "{\n" - " g : {\n" - " x : {\n" - " a : { b : c, e : f },\n" - " q : { x : y }\n" - " },\n" - " y : fff\n" - " }\n" - "}"); + "{\n" + " g : {\n" + " x : {\n" + " a : { b : c, e : f },\n" + " q : { x : y }\n" + " },\n" + " y : fff\n" + " }\n" + "}"); } Y_UNIT_TEST(PrettifyJsonInvalid) { @@ -144,61 +144,61 @@ Y_UNIT_TEST_SUITE(JsonPrettifier) { Y_UNIT_TEST(CompactifyJsonLong) { UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson( - "[\n" - " {\n" - " 'k' : 'v'\n" - " },\n" - " {\n" - " 'a' : 'b'\n" - " }\n" - "]", - true), - "[{k:v},{a:b}]"); + "[\n" + " {\n" + " 'k' : 'v'\n" + " },\n" + " {\n" + " 'a' : 'b'\n" + " }\n" + "]", + true), + "[{k:v},{a:b}]"); UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson( - "{\n" - " 'k' : 'v',\n" - " 'a' : 'b',\n" - " 'x' : [\n" - " 1,\n" - " 2,\n" - " 3\n" - " ]\n" - "}", - true), - "{k:v,a:b,x:[1,2,3]}"); + "{\n" + " 'k' : 'v',\n" + " 'a' : 'b',\n" + " 'x' : [\n" + " 1,\n" + " 2,\n" + " 3\n" + " ]\n" + "}", + true), + "{k:v,a:b,x:[1,2,3]}"); UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson( - "{\n" - " 'k' : 'v',\n" - " 'a' : 'b',\n" - " 'x' : [\n" - " 1,\n" - " {\n" - " 'f' : 'b'\n" - " },\n" - " 3\n" - " ],\n" - " 'm' : 'n'\n" - "}", - true), - "{k:v,a:b,x:[1,{f:b},3],m:n}"); + "{\n" + " 'k' : 'v',\n" + " 'a' : 'b',\n" + " 'x' : [\n" + " 1,\n" + " {\n" + " 'f' : 'b'\n" + " },\n" + " 3\n" + " ],\n" + " 'm' : 'n'\n" + "}", + true), + "{k:v,a:b,x:[1,{f:b},3],m:n}"); UNIT_ASSERT_STRINGS_EQUAL(NJson::CompactifyJson( - "{\n" - " g : {\n" - " x : {\n" - " a : {\n" - " b : c,\n" - " e : f\n" - " },\n" - " q : {\n" - " x : y\n" - " }\n" - " },\n" - " y : fff\n" - " }\n" - "}", - true), - "{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}"); + "{\n" + " g : {\n" + " x : {\n" + " a : {\n" + " b : c,\n" + " e : f\n" + " },\n" + " q : {\n" + " x : y\n" + " }\n" + " },\n" + " y : fff\n" + " }\n" + "}", + true), + "{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}"); } } diff --git a/library/cpp/json/ut/json_reader_fast_ut.cpp b/library/cpp/json/ut/json_reader_fast_ut.cpp index 60dffc91c7..4670975356 100644 --- a/library/cpp/json/ut/json_reader_fast_ut.cpp +++ b/library/cpp/json/ut/json_reader_fast_ut.cpp @@ -7,176 +7,176 @@ #include <util/string/printf.h> namespace NJson { - namespace NTest { - enum ETestEvent { - E_NO_EVENT = 0, - E_ERROR = 1, - E_DICT_OPEN, - E_DICT_CLOSE, - E_ARR_OPEN, - E_ARR_CLOSE, - E_NULL, - E_BOOL, - E_FLT, - E_INT, - E_LONG_LONG, - E_STR, - E_KEY - }; - - struct TEvent { - ETestEvent Type = E_NO_EVENT; - - i64 INum = 0; - double DNum = 0; - TString Str; - - TEvent(ETestEvent e = E_NO_EVENT) - : Type(e) - { - } - - TEvent(double v, ETestEvent e) - : Type(e) - , DNum(v) - { - } - - TEvent(i64 v, ETestEvent e) - : Type(e) - , INum(v) - { - } - - TEvent(TStringBuf t, ETestEvent e) - : Type(e) - , Str(NEscJ::EscapeJ<true, false>(t)) - { - } - - TString ToString() const { - switch (Type) { - default: - return "YOUFAILED"; - case E_ERROR: + namespace NTest { + enum ETestEvent { + E_NO_EVENT = 0, + E_ERROR = 1, + E_DICT_OPEN, + E_DICT_CLOSE, + E_ARR_OPEN, + E_ARR_CLOSE, + E_NULL, + E_BOOL, + E_FLT, + E_INT, + E_LONG_LONG, + E_STR, + E_KEY + }; + + struct TEvent { + ETestEvent Type = E_NO_EVENT; + + i64 INum = 0; + double DNum = 0; + TString Str; + + TEvent(ETestEvent e = E_NO_EVENT) + : Type(e) + { + } + + TEvent(double v, ETestEvent e) + : Type(e) + , DNum(v) + { + } + + TEvent(i64 v, ETestEvent e) + : Type(e) + , INum(v) + { + } + + TEvent(TStringBuf t, ETestEvent e) + : Type(e) + , Str(NEscJ::EscapeJ<true, false>(t)) + { + } + + TString ToString() const { + switch (Type) { + default: + return "YOUFAILED"; + case E_ERROR: return Sprintf("error: %s", Str.data()); - case E_DICT_OPEN: - return "{"; - case E_DICT_CLOSE: - return "}"; - case E_ARR_OPEN: - return "["; - case E_ARR_CLOSE: - return "]"; - case E_NULL: - return "null"; - case E_BOOL: - return INum ? "true" : "false"; - case E_INT: - return ::ToString(INum); - case E_FLT: - return ::ToString(DNum); - case E_STR: + case E_DICT_OPEN: + return "{"; + case E_DICT_CLOSE: + return "}"; + case E_ARR_OPEN: + return "["; + case E_ARR_CLOSE: + return "]"; + case E_NULL: + return "null"; + case E_BOOL: + return INum ? "true" : "false"; + case E_INT: + return ::ToString(INum); + case E_FLT: + return ::ToString(DNum); + case E_STR: return Sprintf("%s", Str.data()); - case E_KEY: + case E_KEY: return Sprintf("key: %s", Str.data()); - } - } - }; - - using TEvents = TVector<TEvent>; - - struct TTestHandler : TJsonCallbacks { - TEvents Events; - - bool OnOpenMap() override { - Events.push_back(E_DICT_OPEN); - return true; - } - - bool OnCloseMap() override { - Events.push_back(E_DICT_CLOSE); - return true; - } - - bool OnOpenArray() override { - Events.push_back(E_ARR_OPEN); - return true; - } - - bool OnCloseArray() override { - Events.push_back(E_ARR_CLOSE); - return true; - } - - bool OnNull() override { - Events.push_back(E_NULL); - return true; - } - - bool OnBoolean(bool v) override { - Events.push_back(TEvent((i64)v, E_BOOL)); - return true; - } - - bool OnInteger(long long v) override { - Events.push_back(TEvent((i64)v, E_INT)); - return true; - } - - bool OnUInteger(unsigned long long v) override { - return OnInteger(v); - } - - bool OnDouble(double v) override { - Events.push_back(TEvent(v, E_FLT)); - return true; - } - - bool OnString(const TStringBuf& v) override { - Events.push_back(TEvent(v, E_STR)); - return true; - } - - bool OnMapKey(const TStringBuf& v) override { - Events.push_back(TEvent(v, E_KEY)); - return true; - } - - void OnError(size_t, TStringBuf token) override { - Events.push_back(TEvent(token, E_ERROR)); - } - - void Assert(const TEvents& e, TString str) { - try { - UNIT_ASSERT_VALUES_EQUAL_C(e.size(), Events.size(), str); - - for (ui32 i = 0, sz = e.size(); i < sz; ++i) { + } + } + }; + + using TEvents = TVector<TEvent>; + + struct TTestHandler : TJsonCallbacks { + TEvents Events; + + bool OnOpenMap() override { + Events.push_back(E_DICT_OPEN); + return true; + } + + bool OnCloseMap() override { + Events.push_back(E_DICT_CLOSE); + return true; + } + + bool OnOpenArray() override { + Events.push_back(E_ARR_OPEN); + return true; + } + + bool OnCloseArray() override { + Events.push_back(E_ARR_CLOSE); + return true; + } + + bool OnNull() override { + Events.push_back(E_NULL); + return true; + } + + bool OnBoolean(bool v) override { + Events.push_back(TEvent((i64)v, E_BOOL)); + return true; + } + + bool OnInteger(long long v) override { + Events.push_back(TEvent((i64)v, E_INT)); + return true; + } + + bool OnUInteger(unsigned long long v) override { + return OnInteger(v); + } + + bool OnDouble(double v) override { + Events.push_back(TEvent(v, E_FLT)); + return true; + } + + bool OnString(const TStringBuf& v) override { + Events.push_back(TEvent(v, E_STR)); + return true; + } + + bool OnMapKey(const TStringBuf& v) override { + Events.push_back(TEvent(v, E_KEY)); + return true; + } + + void OnError(size_t, TStringBuf token) override { + Events.push_back(TEvent(token, E_ERROR)); + } + + void Assert(const TEvents& e, TString str) { + try { + UNIT_ASSERT_VALUES_EQUAL_C(e.size(), Events.size(), str); + + for (ui32 i = 0, sz = e.size(); i < sz; ++i) { UNIT_ASSERT_VALUES_EQUAL_C((int)e[i].Type, (int)Events[i].Type, Sprintf("'%s' %u", str.data(), i)); UNIT_ASSERT_VALUES_EQUAL_C(e[i].INum, Events[i].INum, Sprintf("'%s' %u", str.data(), i)); UNIT_ASSERT_VALUES_EQUAL_C(e[i].DNum, Events[i].DNum, Sprintf("'%s' %u", str.data(), i)); UNIT_ASSERT_VALUES_EQUAL_C(e[i].Str, Events[i].Str, Sprintf("'%s' %u", str.data(), i)); - } - } catch (const yexception&) { - Clog << "Exception at '" << str << "'" << Endl; - for (const auto& event : Events) { - Clog << event.ToString() << Endl; - } - - throw; - } + } + } catch (const yexception&) { + Clog << "Exception at '" << str << "'" << Endl; + for (const auto& event : Events) { + Clog << event.ToString() << Endl; + } + + throw; + } } - }; + }; } } -class TFastJsonTest: public TTestBase { - UNIT_TEST_SUITE(TFastJsonTest) +class TFastJsonTest: public TTestBase { + UNIT_TEST_SUITE(TFastJsonTest) UNIT_TEST(TestParse) UNIT_TEST(TestReadJsonFastTree) UNIT_TEST(TestNoInlineComment) - UNIT_TEST_SUITE_END(); - + UNIT_TEST_SUITE_END(); + public: template <bool accept> void DoTestParse(TStringBuf json, ui32 amount, ...) { @@ -188,45 +188,45 @@ public: ETestEvent e = (ETestEvent)va_arg(vl, int); switch ((int)e) { - case E_NO_EVENT: - case E_DICT_OPEN: - case E_DICT_CLOSE: - case E_ARR_OPEN: - case E_ARR_CLOSE: - case E_NULL: - evs.push_back(e); - break; - case E_BOOL: { - bool v = va_arg(vl, int); - evs.push_back(TEvent((i64)v, E_BOOL)); - break; - } - case E_INT: { - i64 i = va_arg(vl, int); - evs.push_back(TEvent(i, E_INT)); - break; - } - case E_LONG_LONG: { - i64 i = va_arg(vl, long long); - evs.push_back(TEvent(i, E_INT)); - break; - } - case E_FLT: { - double f = va_arg(vl, double); - evs.push_back(TEvent(f, E_FLT)); - break; - } - case E_STR: { - const char* s = va_arg(vl, const char*); - evs.push_back(TEvent(TStringBuf(s), E_STR)); - break; - } - case E_KEY: - case E_ERROR: { - const char* s = va_arg(vl, const char*); - evs.push_back(TEvent(TStringBuf(s), e)); - break; - } + case E_NO_EVENT: + case E_DICT_OPEN: + case E_DICT_CLOSE: + case E_ARR_OPEN: + case E_ARR_CLOSE: + case E_NULL: + evs.push_back(e); + break; + case E_BOOL: { + bool v = va_arg(vl, int); + evs.push_back(TEvent((i64)v, E_BOOL)); + break; + } + case E_INT: { + i64 i = va_arg(vl, int); + evs.push_back(TEvent(i, E_INT)); + break; + } + case E_LONG_LONG: { + i64 i = va_arg(vl, long long); + evs.push_back(TEvent(i, E_INT)); + break; + } + case E_FLT: { + double f = va_arg(vl, double); + evs.push_back(TEvent(f, E_FLT)); + break; + } + case E_STR: { + const char* s = va_arg(vl, const char*); + evs.push_back(TEvent(TStringBuf(s), E_STR)); + break; + } + case E_KEY: + case E_ERROR: { + const char* s = va_arg(vl, const char*); + evs.push_back(TEvent(TStringBuf(s), e)); + break; + } } } va_end(vl); @@ -258,25 +258,25 @@ public: DoTestParse<false>("[a,,b]", 3, E_ARR_OPEN, E_STR, "a", E_ERROR, "invalid syntax at token: ','"); DoTestParse<true>("{ k : v }", 4, E_DICT_OPEN, E_KEY, "k", E_STR, "v", E_DICT_CLOSE); DoTestParse<true>("{a:'\\b'/*comment*/, k /*comment*/\n : v }", 6, E_DICT_OPEN, E_KEY, "a", E_STR, "\b", E_KEY, "k", E_STR, "v", E_DICT_CLOSE); - DoTestParse<true>("{a:.15, k : v }", 6, E_DICT_OPEN, E_KEY, "a", E_FLT, .15, E_KEY, "k", E_STR, "v", E_DICT_CLOSE); - DoTestParse<true>("[ a, -.1e+5, 1E-7]", 5, E_ARR_OPEN, E_STR, "a", E_FLT, -.1e+5, E_FLT, 1e-7, E_ARR_CLOSE); + DoTestParse<true>("{a:.15, k : v }", 6, E_DICT_OPEN, E_KEY, "a", E_FLT, .15, E_KEY, "k", E_STR, "v", E_DICT_CLOSE); + DoTestParse<true>("[ a, -.1e+5, 1E-7]", 5, E_ARR_OPEN, E_STR, "a", E_FLT, -.1e+5, E_FLT, 1e-7, E_ARR_CLOSE); DoTestParse<true>("{}", 2, E_DICT_OPEN, E_DICT_CLOSE); - DoTestParse<true>("{ a : x, b : [ c, d, ] }", 9, E_DICT_OPEN, E_KEY, "a", E_STR, "x", E_KEY, "b", E_ARR_OPEN, E_STR, "c", E_STR, "d", E_ARR_CLOSE, E_DICT_CLOSE); - DoTestParse<false>("{ a : x, b : [ c, d,, ] }", 8, E_DICT_OPEN, E_KEY, "a", E_STR, "x", E_KEY, "b", E_ARR_OPEN, E_STR, "c", E_STR, "d", E_ERROR, "invalid syntax at token: ','"); - // DoTestParse<false>("{ a : x : y }", 4, E_DICT_OPEN - // , E_KEY, "a", E_STR, "x" - // , E_ERROR - // , ":"); - // DoTestParse<false>("{queries:{ref:[]},{nonref:[]}}", 8, E_DICT_OPEN - // , E_KEY, "queries", E_DICT_OPEN - // , E_KEY, "ref", E_ARR_OPEN, E_ARR_CLOSE - // , E_DICT_CLOSE, E_ERROR, ""); + DoTestParse<true>("{ a : x, b : [ c, d, ] }", 9, E_DICT_OPEN, E_KEY, "a", E_STR, "x", E_KEY, "b", E_ARR_OPEN, E_STR, "c", E_STR, "d", E_ARR_CLOSE, E_DICT_CLOSE); + DoTestParse<false>("{ a : x, b : [ c, d,, ] }", 8, E_DICT_OPEN, E_KEY, "a", E_STR, "x", E_KEY, "b", E_ARR_OPEN, E_STR, "c", E_STR, "d", E_ERROR, "invalid syntax at token: ','"); + // DoTestParse<false>("{ a : x : y }", 4, E_DICT_OPEN + // , E_KEY, "a", E_STR, "x" + // , E_ERROR + // , ":"); + // DoTestParse<false>("{queries:{ref:[]},{nonref:[]}}", 8, E_DICT_OPEN + // , E_KEY, "queries", E_DICT_OPEN + // , E_KEY, "ref", E_ARR_OPEN, E_ARR_CLOSE + // , E_DICT_CLOSE, E_ERROR, ""); DoTestParse<true>("'100x00'", 1, E_STR, "100x00"); DoTestParse<true>("-1", 1, E_INT, -1); DoTestParse<true>("-9223372036854775808", 1, E_LONG_LONG, (long long)Min<i64>()); DoTestParse<false>("100x00", 1, E_ERROR, "invalid syntax at token: '100x'"); DoTestParse<false>("100 200", 2, E_INT, 100, E_ERROR, "invalid syntax at token: '200'"); - DoTestParse<true>("{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}", 22, E_DICT_OPEN, E_KEY, "g", E_DICT_OPEN, E_KEY, "x", E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_KEY, "b", E_STR, "c", E_KEY, "e", E_STR, "f", E_DICT_CLOSE, E_KEY, "q", E_DICT_OPEN, E_KEY, "x", E_STR, "y", E_DICT_CLOSE, E_DICT_CLOSE, E_KEY, "y", E_STR, "fff", E_DICT_CLOSE, E_DICT_CLOSE); + DoTestParse<true>("{g:{x:{a:{b:c,e:f},q:{x:y}},y:fff}}", 22, E_DICT_OPEN, E_KEY, "g", E_DICT_OPEN, E_KEY, "x", E_DICT_OPEN, E_KEY, "a", E_DICT_OPEN, E_KEY, "b", E_STR, "c", E_KEY, "e", E_STR, "f", E_DICT_CLOSE, E_KEY, "q", E_DICT_OPEN, E_KEY, "x", E_STR, "y", E_DICT_CLOSE, E_DICT_CLOSE, E_KEY, "y", E_STR, "fff", E_DICT_CLOSE, E_DICT_CLOSE); } void TestReadJsonFastTree() { diff --git a/library/cpp/json/ut/json_reader_ut.cpp b/library/cpp/json/ut/json_reader_ut.cpp index cd31afa0b8..958ed19730 100644 --- a/library/cpp/json/ut/json_reader_ut.cpp +++ b/library/cpp/json/ut/json_reader_ut.cpp @@ -7,12 +7,12 @@ using namespace NJson; class TReformatCallbacks: public TJsonCallbacks { - TJsonWriter& Writer; - + TJsonWriter& Writer; + public: - TReformatCallbacks(TJsonWriter& writer) - : Writer(writer) - { + TReformatCallbacks(TJsonWriter& writer) + : Writer(writer) + { } bool OnBoolean(bool val) override { @@ -30,7 +30,7 @@ public: return true; } - bool OnString(const TStringBuf& val) override { + bool OnString(const TStringBuf& val) override { Writer.Write(val); return true; } @@ -60,7 +60,7 @@ public: return true; } - bool OnMapKey(const TStringBuf& val) override { + bool OnMapKey(const TStringBuf& val) override { Writer.Write(val); return true; } @@ -145,7 +145,7 @@ Y_UNIT_TEST_SUITE(TJsonReaderTest) { UNIT_ASSERT_VALUES_EQUAL(value["array"].GetStringRobust(), "[1,2,3,\"TString\"]"); UNIT_ASSERT_VALUES_EQUAL(value["null value"].GetStringRobust(), "null"); - const TJsonValue::TArray* array; + const TJsonValue::TArray* array; UNIT_ASSERT(GetArrayPointer(value, "array", &array)); UNIT_ASSERT_VALUES_EQUAL(value["array"].GetArray().size(), array->size()); UNIT_ASSERT_VALUES_EQUAL(value["array"][0].GetInteger(), (*array)[0].GetInteger()); @@ -353,7 +353,7 @@ Y_UNIT_TEST_SUITE(TJsonReaderTest) { UNIT_ASSERT_EQUAL(value["test"].GetDouble(), 0.0); UNIT_ASSERT_EQUAL(value["test"].GetDoubleRobust(), static_cast<double>(Max<ui64>())); } // Max<ui64>() - } // TJsonDoubleTest + } // TJsonDoubleTest Y_UNIT_TEST(TJsonInvalidTest) { { @@ -397,34 +397,34 @@ Y_UNIT_TEST_SUITE(TJsonReaderTest) { UNIT_ASSERT_VALUES_EQUAL("", v.GetMap().begin()->second.GetString()); } } - - -static const TString YANDEX_STREAMING_JSON("{\"a\":1}//d{\"b\":2}"); - - -Y_UNIT_TEST_SUITE(TCompareReadJsonFast) { - Y_UNIT_TEST(NoEndl) { - NJson::TJsonValue parsed; - - bool success = NJson::ReadJsonTree(YANDEX_STREAMING_JSON, &parsed, false); - bool fast_success = NJson::ReadJsonFastTree(YANDEX_STREAMING_JSON, &parsed, false); - UNIT_ASSERT(success == fast_success); - } - Y_UNIT_TEST(WithEndl) { - NJson::TJsonValue parsed1; - NJson::TJsonValue parsed2; - - bool success = NJson::ReadJsonTree(YANDEX_STREAMING_JSON + "\n", &parsed1, false); - bool fast_success = NJson::ReadJsonFastTree(YANDEX_STREAMING_JSON + "\n", &parsed2, false); - - UNIT_ASSERT_VALUES_EQUAL(success, fast_success); - } - Y_UNIT_TEST(NoQuotes) { - TString streamingJson = "{a:1}"; - NJson::TJsonValue parsed; - - bool success = NJson::ReadJsonTree(streamingJson, &parsed, false); - bool fast_success = NJson::ReadJsonFastTree(streamingJson, &parsed, false); - UNIT_ASSERT(success != fast_success); - } -} + + +static const TString YANDEX_STREAMING_JSON("{\"a\":1}//d{\"b\":2}"); + + +Y_UNIT_TEST_SUITE(TCompareReadJsonFast) { + Y_UNIT_TEST(NoEndl) { + NJson::TJsonValue parsed; + + bool success = NJson::ReadJsonTree(YANDEX_STREAMING_JSON, &parsed, false); + bool fast_success = NJson::ReadJsonFastTree(YANDEX_STREAMING_JSON, &parsed, false); + UNIT_ASSERT(success == fast_success); + } + Y_UNIT_TEST(WithEndl) { + NJson::TJsonValue parsed1; + NJson::TJsonValue parsed2; + + bool success = NJson::ReadJsonTree(YANDEX_STREAMING_JSON + "\n", &parsed1, false); + bool fast_success = NJson::ReadJsonFastTree(YANDEX_STREAMING_JSON + "\n", &parsed2, false); + + UNIT_ASSERT_VALUES_EQUAL(success, fast_success); + } + Y_UNIT_TEST(NoQuotes) { + TString streamingJson = "{a:1}"; + NJson::TJsonValue parsed; + + bool success = NJson::ReadJsonTree(streamingJson, &parsed, false); + bool fast_success = NJson::ReadJsonFastTree(streamingJson, &parsed, false); + UNIT_ASSERT(success != fast_success); + } +} diff --git a/library/cpp/json/ut/json_writer_ut.cpp b/library/cpp/json/ut/json_writer_ut.cpp index ca11d34dad..0fe747d704 100644 --- a/library/cpp/json/ut/json_writer_ut.cpp +++ b/library/cpp/json/ut/json_writer_ut.cpp @@ -8,7 +8,7 @@ using namespace NJson; Y_UNIT_TEST_SUITE(TJsonWriterTest) { Y_UNIT_TEST(SimpleWriteTest) { TString expected1 = "{\"key1\":1,\"key2\":2,\"key3\":3"; - TString expected2 = expected1 + ",\"array\":[\"stroka\",false]"; + TString expected2 = expected1 + ",\"array\":[\"stroka\",false]"; TString expected3 = expected2 + "}"; TStringStream out; @@ -120,7 +120,7 @@ Y_UNIT_TEST_SUITE(TJsonWriterTest) { WriteJson(&out, &v); UNIT_ASSERT_VALUES_EQUAL(out.Str(), expected); } // 18446744073709551615 - } // SimpleUnsignedIntegerWriteTest + } // SimpleUnsignedIntegerWriteTest Y_UNIT_TEST(WriteOptionalTest) { { diff --git a/library/cpp/json/ut/ya.make b/library/cpp/json/ut/ya.make index 8e0362d84b..5f3a2d8510 100644 --- a/library/cpp/json/ut/ya.make +++ b/library/cpp/json/ut/ya.make @@ -2,10 +2,10 @@ OWNER(velavokr) UNITTEST_FOR(library/cpp/json) -PEERDIR( +PEERDIR( library/cpp/string_utils/relaxed_escaper -) - +) + SRCS( json_reader_fast_ut.cpp json_reader_ut.cpp diff --git a/library/cpp/json/writer/json.cpp b/library/cpp/json/writer/json.cpp index 02370c2d79..6bad7cf3a9 100644 --- a/library/cpp/json/writer/json.cpp +++ b/library/cpp/json/writer/json.cpp @@ -1,94 +1,94 @@ #include "json.h" - + #include <library/cpp/json/json_value.h> - + #include <util/string/cast.h> -#include <util/string/strspn.h> +#include <util/string/strspn.h> #include <util/generic/algorithm.h> #include <util/generic/ymath.h> -#include <util/generic/singleton.h> +#include <util/generic/singleton.h> namespace NJsonWriter { - TBuf::TBuf(EHtmlEscapeMode mode, IOutputStream* stream) - : Stream(stream) - , NeedComma(false) - , NeedNewline(false) - , EscapeMode(mode) - , IndentSpaces(0) - , WriteNanAsString(false) - { - Y_ASSERT(mode == HEM_DONT_ESCAPE_HTML || - mode == HEM_ESCAPE_HTML || - mode == HEM_RELAXED || - mode == HEM_UNSAFE); - if (!Stream) { - StringStream.Reset(new TStringStream); - Stream = StringStream.Get(); - } - - Stack.reserve(64); // should be enough for most cases - StackPush(JE_OUTER_SPACE); - } - - static const char* EntityToStr(EJsonEntity e) { - switch (e) { - case JE_OUTER_SPACE: - return "JE_OUTER_SPACE"; - case JE_LIST: - return "JE_LIST"; - case JE_OBJECT: - return "JE_OBJECT"; - case JE_PAIR: - return "JE_PAIR"; - default: - return "JE_unknown"; - } - } - - inline void TBuf::StackPush(EJsonEntity e) { - Stack.push_back(e); - } - - inline EJsonEntity TBuf::StackTop() const { - return Stack.back(); - } - - inline void TBuf::StackPop() { - Y_ASSERT(!Stack.empty()); - const EJsonEntity current = StackTop(); - Stack.pop_back(); - switch (current) { - case JE_OUTER_SPACE: - ythrow TError() << "JSON writer: stack empty"; - case JE_LIST: - PrintIndentation(true); - RawWriteChar(']'); - break; - case JE_OBJECT: - PrintIndentation(true); - RawWriteChar('}'); - break; - case JE_PAIR: - break; - } - NeedComma = true; - NeedNewline = true; - } - - inline void TBuf::CheckAndPop(EJsonEntity e) { - if (Y_UNLIKELY(StackTop() != e)) { - ythrow TError() << "JSON writer: unexpected value " - << EntityToStr(StackTop()) << " on the stack"; - } - StackPop(); - } - - void TBuf::PrintIndentation(bool closing) { - if (!IndentSpaces) - return; - const int indentation = IndentSpaces * (Stack.size() - 1); - if (!indentation && !closing) - return; + TBuf::TBuf(EHtmlEscapeMode mode, IOutputStream* stream) + : Stream(stream) + , NeedComma(false) + , NeedNewline(false) + , EscapeMode(mode) + , IndentSpaces(0) + , WriteNanAsString(false) + { + Y_ASSERT(mode == HEM_DONT_ESCAPE_HTML || + mode == HEM_ESCAPE_HTML || + mode == HEM_RELAXED || + mode == HEM_UNSAFE); + if (!Stream) { + StringStream.Reset(new TStringStream); + Stream = StringStream.Get(); + } + + Stack.reserve(64); // should be enough for most cases + StackPush(JE_OUTER_SPACE); + } + + static const char* EntityToStr(EJsonEntity e) { + switch (e) { + case JE_OUTER_SPACE: + return "JE_OUTER_SPACE"; + case JE_LIST: + return "JE_LIST"; + case JE_OBJECT: + return "JE_OBJECT"; + case JE_PAIR: + return "JE_PAIR"; + default: + return "JE_unknown"; + } + } + + inline void TBuf::StackPush(EJsonEntity e) { + Stack.push_back(e); + } + + inline EJsonEntity TBuf::StackTop() const { + return Stack.back(); + } + + inline void TBuf::StackPop() { + Y_ASSERT(!Stack.empty()); + const EJsonEntity current = StackTop(); + Stack.pop_back(); + switch (current) { + case JE_OUTER_SPACE: + ythrow TError() << "JSON writer: stack empty"; + case JE_LIST: + PrintIndentation(true); + RawWriteChar(']'); + break; + case JE_OBJECT: + PrintIndentation(true); + RawWriteChar('}'); + break; + case JE_PAIR: + break; + } + NeedComma = true; + NeedNewline = true; + } + + inline void TBuf::CheckAndPop(EJsonEntity e) { + if (Y_UNLIKELY(StackTop() != e)) { + ythrow TError() << "JSON writer: unexpected value " + << EntityToStr(StackTop()) << " on the stack"; + } + StackPop(); + } + + void TBuf::PrintIndentation(bool closing) { + if (!IndentSpaces) + return; + const int indentation = IndentSpaces * (Stack.size() - 1); + if (!indentation && !closing) + return; PrintWhitespaces(Max(0, indentation), true); } @@ -107,258 +107,258 @@ namespace NJsonWriter { } while (count > 0); } - inline void TBuf::WriteComma() { - if (NeedComma) { - RawWriteChar(','); - } - NeedComma = true; - - if (NeedNewline) { - PrintIndentation(false); - } - NeedNewline = true; - } - - inline void TBuf::BeginValue() { - if (Y_UNLIKELY(KeyExpected())) { - ythrow TError() << "JSON writer: value written, " - "but expected a key:value pair"; - } - WriteComma(); - } - - inline void TBuf::BeginKey() { - if (Y_UNLIKELY(!KeyExpected())) { - ythrow TError() << "JSON writer: key written outside of an object"; - } - WriteComma(); - StackPush(JE_PAIR); - NeedComma = false; - NeedNewline = false; - } - - inline void TBuf::EndValue() { - if (StackTop() == JE_PAIR) { - StackPop(); - } - } - - TValueContext TBuf::BeginList() { - NeedNewline = true; - BeginValue(); - RawWriteChar('['); - StackPush(JE_LIST); - NeedComma = false; - return TValueContext(*this); - } - - TPairContext TBuf::BeginObject() { - NeedNewline = true; - BeginValue(); - RawWriteChar('{'); - StackPush(JE_OBJECT); - NeedComma = false; - return TPairContext(*this); - } - - TAfterColonContext TBuf::UnsafeWriteKey(const TStringBuf& s) { - BeginKey(); - RawWriteChar('"'); - UnsafeWriteRawBytes(s); - UnsafeWriteRawBytes("\":", 2); - return TAfterColonContext(*this); - } - - TAfterColonContext TBuf::WriteKey(const TStringBuf& s) { - // use the default escaping mode for this object - return WriteKey(s, EscapeMode); - } - - TAfterColonContext TBuf::WriteKey(const TStringBuf& s, EHtmlEscapeMode hem) { - BeginKey(); - WriteBareString(s, hem); - RawWriteChar(':'); - return TAfterColonContext(*this); - } - - TAfterColonContext TBuf::CompatWriteKeyWithoutQuotes(const TStringBuf& s) { - BeginKey(); - Y_ASSERT(AllOf(s, [](char x) { return 'a' <= x && x <= 'z'; })); - UnsafeWriteRawBytes(s); - RawWriteChar(':'); - return TAfterColonContext(*this); - } - - TBuf& TBuf::EndList() { - CheckAndPop(JE_LIST); - EndValue(); - return *this; - } - - TBuf& TBuf::EndObject() { - CheckAndPop(JE_OBJECT); - EndValue(); - return *this; - } - - TValueContext TBuf::WriteString(const TStringBuf& s) { - // use the default escaping mode for this object - return WriteString(s, EscapeMode); - } - - TValueContext TBuf::WriteString(const TStringBuf& s, EHtmlEscapeMode hem) { - BeginValue(); - WriteBareString(s, hem); - EndValue(); - return TValueContext(*this); - } - - TValueContext TBuf::WriteNull() { + inline void TBuf::WriteComma() { + if (NeedComma) { + RawWriteChar(','); + } + NeedComma = true; + + if (NeedNewline) { + PrintIndentation(false); + } + NeedNewline = true; + } + + inline void TBuf::BeginValue() { + if (Y_UNLIKELY(KeyExpected())) { + ythrow TError() << "JSON writer: value written, " + "but expected a key:value pair"; + } + WriteComma(); + } + + inline void TBuf::BeginKey() { + if (Y_UNLIKELY(!KeyExpected())) { + ythrow TError() << "JSON writer: key written outside of an object"; + } + WriteComma(); + StackPush(JE_PAIR); + NeedComma = false; + NeedNewline = false; + } + + inline void TBuf::EndValue() { + if (StackTop() == JE_PAIR) { + StackPop(); + } + } + + TValueContext TBuf::BeginList() { + NeedNewline = true; + BeginValue(); + RawWriteChar('['); + StackPush(JE_LIST); + NeedComma = false; + return TValueContext(*this); + } + + TPairContext TBuf::BeginObject() { + NeedNewline = true; + BeginValue(); + RawWriteChar('{'); + StackPush(JE_OBJECT); + NeedComma = false; + return TPairContext(*this); + } + + TAfterColonContext TBuf::UnsafeWriteKey(const TStringBuf& s) { + BeginKey(); + RawWriteChar('"'); + UnsafeWriteRawBytes(s); + UnsafeWriteRawBytes("\":", 2); + return TAfterColonContext(*this); + } + + TAfterColonContext TBuf::WriteKey(const TStringBuf& s) { + // use the default escaping mode for this object + return WriteKey(s, EscapeMode); + } + + TAfterColonContext TBuf::WriteKey(const TStringBuf& s, EHtmlEscapeMode hem) { + BeginKey(); + WriteBareString(s, hem); + RawWriteChar(':'); + return TAfterColonContext(*this); + } + + TAfterColonContext TBuf::CompatWriteKeyWithoutQuotes(const TStringBuf& s) { + BeginKey(); + Y_ASSERT(AllOf(s, [](char x) { return 'a' <= x && x <= 'z'; })); + UnsafeWriteRawBytes(s); + RawWriteChar(':'); + return TAfterColonContext(*this); + } + + TBuf& TBuf::EndList() { + CheckAndPop(JE_LIST); + EndValue(); + return *this; + } + + TBuf& TBuf::EndObject() { + CheckAndPop(JE_OBJECT); + EndValue(); + return *this; + } + + TValueContext TBuf::WriteString(const TStringBuf& s) { + // use the default escaping mode for this object + return WriteString(s, EscapeMode); + } + + TValueContext TBuf::WriteString(const TStringBuf& s, EHtmlEscapeMode hem) { + BeginValue(); + WriteBareString(s, hem); + EndValue(); + return TValueContext(*this); + } + + TValueContext TBuf::WriteNull() { UnsafeWriteValue(TStringBuf("null")); - return TValueContext(*this); - } + return TValueContext(*this); + } - TValueContext TBuf::WriteBool(bool b) { + TValueContext TBuf::WriteBool(bool b) { constexpr TStringBuf trueVal = "true"; constexpr TStringBuf falseVal = "false"; UnsafeWriteValue(b ? trueVal : falseVal); - return TValueContext(*this); - } - - TValueContext TBuf::WriteInt(int i) { - char buf[22]; // enough to hold any 64-bit number - size_t len = ToString(i, buf, sizeof(buf)); - UnsafeWriteValue(buf, len); - return TValueContext(*this); - } - - TValueContext TBuf::WriteLongLong(long long i) { - static_assert(sizeof(long long) <= 8, "expect sizeof(long long) <= 8"); - char buf[22]; // enough to hold any 64-bit number - size_t len = ToString(i, buf, sizeof(buf)); - UnsafeWriteValue(buf, len); - return TValueContext(*this); - } - - TValueContext TBuf::WriteULongLong(unsigned long long i) { - char buf[22]; // enough to hold any 64-bit number - size_t len = ToString(i, buf, sizeof(buf)); - UnsafeWriteValue(buf, len); - return TValueContext(*this); - } - - template <class TFloat> - TValueContext TBuf::WriteFloatImpl(TFloat f, EFloatToStringMode mode, int ndigits) { - char buf[512]; // enough to hold most floats, the same buffer is used in FloatToString implementation - if (Y_UNLIKELY(!IsValidFloat(f))) { - if (WriteNanAsString) { - const size_t size = FloatToString(f, buf, Y_ARRAY_SIZE(buf)); - WriteString(TStringBuf(buf, size)); - return TValueContext(*this); - } else { - ythrow TError() << "JSON writer: invalid float value: " << FloatToString(f); - } + return TValueContext(*this); + } + + TValueContext TBuf::WriteInt(int i) { + char buf[22]; // enough to hold any 64-bit number + size_t len = ToString(i, buf, sizeof(buf)); + UnsafeWriteValue(buf, len); + return TValueContext(*this); + } + + TValueContext TBuf::WriteLongLong(long long i) { + static_assert(sizeof(long long) <= 8, "expect sizeof(long long) <= 8"); + char buf[22]; // enough to hold any 64-bit number + size_t len = ToString(i, buf, sizeof(buf)); + UnsafeWriteValue(buf, len); + return TValueContext(*this); + } + + TValueContext TBuf::WriteULongLong(unsigned long long i) { + char buf[22]; // enough to hold any 64-bit number + size_t len = ToString(i, buf, sizeof(buf)); + UnsafeWriteValue(buf, len); + return TValueContext(*this); + } + + template <class TFloat> + TValueContext TBuf::WriteFloatImpl(TFloat f, EFloatToStringMode mode, int ndigits) { + char buf[512]; // enough to hold most floats, the same buffer is used in FloatToString implementation + if (Y_UNLIKELY(!IsValidFloat(f))) { + if (WriteNanAsString) { + const size_t size = FloatToString(f, buf, Y_ARRAY_SIZE(buf)); + WriteString(TStringBuf(buf, size)); + return TValueContext(*this); + } else { + ythrow TError() << "JSON writer: invalid float value: " << FloatToString(f); + } } - size_t len = FloatToString(f, buf, Y_ARRAY_SIZE(buf), mode, ndigits); - UnsafeWriteValue(buf, len); - return TValueContext(*this); - } - - TValueContext TBuf::WriteFloat(float f, EFloatToStringMode mode, int ndigits) { - return WriteFloatImpl(f, mode, ndigits); - } - - TValueContext TBuf::WriteDouble(double f, EFloatToStringMode mode, int ndigits) { - return WriteFloatImpl(f, mode, ndigits); - } - - namespace { - struct TFinder: public TCompactStrSpn { - inline TFinder() - : TCompactStrSpn("\xe2\\\"\b\n\f\r\t<>&\'/") - { - for (ui8 ch = 0; ch < 0x20; ++ch) { - Set(ch); - } - } - }; - } - - inline void TBuf::WriteBareString(const TStringBuf s, EHtmlEscapeMode hem) { - RawWriteChar('"'); - const auto& specialChars = *Singleton<TFinder>(); - const char* b = s.begin(); - const char* e = s.end(); - const char* i = b; - while ((i = specialChars.FindFirstOf(i, e)) != e) { - // U+2028 (line separator) and U+2029 (paragraph separator) are valid string - // contents in JSON, but are treated as line breaks in JavaScript, breaking JSONP. - // In UTF-8, U+2028 is "\xe2\x80\xa8" and U+2029 is "\xe2\x80\xa9". - if (Y_UNLIKELY(e - i >= 3 && i[0] == '\xe2' && i[1] == '\x80' && (i[2] | 1) == '\xa9')) { - UnsafeWriteRawBytes(b, i - b); - UnsafeWriteRawBytes(i[2] == '\xa9' ? "\\u2029" : "\\u2028", 6); - b = i = i + 3; - } else if (EscapedWriteChar(b, i, hem)) { - b = ++i; - } else { - ++i; - } - } - UnsafeWriteRawBytes(b, e - b); - RawWriteChar('"'); - } - - inline void TBuf::RawWriteChar(char c) { - Stream->Write(c); - } - - void TBuf::WriteHexEscape(unsigned char c) { - Y_ASSERT(c < 0x80); - UnsafeWriteRawBytes("\\u00", 4); - static const char hexDigits[] = "0123456789ABCDEF"; - RawWriteChar(hexDigits[(c & 0xf0) >> 4]); - RawWriteChar(hexDigits[(c & 0x0f)]); - } - -#define MATCH(sym, string) \ - case sym: \ - UnsafeWriteRawBytes(beg, cur - beg); \ + size_t len = FloatToString(f, buf, Y_ARRAY_SIZE(buf), mode, ndigits); + UnsafeWriteValue(buf, len); + return TValueContext(*this); + } + + TValueContext TBuf::WriteFloat(float f, EFloatToStringMode mode, int ndigits) { + return WriteFloatImpl(f, mode, ndigits); + } + + TValueContext TBuf::WriteDouble(double f, EFloatToStringMode mode, int ndigits) { + return WriteFloatImpl(f, mode, ndigits); + } + + namespace { + struct TFinder: public TCompactStrSpn { + inline TFinder() + : TCompactStrSpn("\xe2\\\"\b\n\f\r\t<>&\'/") + { + for (ui8 ch = 0; ch < 0x20; ++ch) { + Set(ch); + } + } + }; + } + + inline void TBuf::WriteBareString(const TStringBuf s, EHtmlEscapeMode hem) { + RawWriteChar('"'); + const auto& specialChars = *Singleton<TFinder>(); + const char* b = s.begin(); + const char* e = s.end(); + const char* i = b; + while ((i = specialChars.FindFirstOf(i, e)) != e) { + // U+2028 (line separator) and U+2029 (paragraph separator) are valid string + // contents in JSON, but are treated as line breaks in JavaScript, breaking JSONP. + // In UTF-8, U+2028 is "\xe2\x80\xa8" and U+2029 is "\xe2\x80\xa9". + if (Y_UNLIKELY(e - i >= 3 && i[0] == '\xe2' && i[1] == '\x80' && (i[2] | 1) == '\xa9')) { + UnsafeWriteRawBytes(b, i - b); + UnsafeWriteRawBytes(i[2] == '\xa9' ? "\\u2029" : "\\u2028", 6); + b = i = i + 3; + } else if (EscapedWriteChar(b, i, hem)) { + b = ++i; + } else { + ++i; + } + } + UnsafeWriteRawBytes(b, e - b); + RawWriteChar('"'); + } + + inline void TBuf::RawWriteChar(char c) { + Stream->Write(c); + } + + void TBuf::WriteHexEscape(unsigned char c) { + Y_ASSERT(c < 0x80); + UnsafeWriteRawBytes("\\u00", 4); + static const char hexDigits[] = "0123456789ABCDEF"; + RawWriteChar(hexDigits[(c & 0xf0) >> 4]); + RawWriteChar(hexDigits[(c & 0x0f)]); + } + +#define MATCH(sym, string) \ + case sym: \ + UnsafeWriteRawBytes(beg, cur - beg); \ UnsafeWriteRawBytes(TStringBuf(string)); \ - return true - - inline bool TBuf::EscapedWriteChar(const char* beg, const char* cur, EHtmlEscapeMode hem) { - unsigned char c = *cur; - if (hem == HEM_ESCAPE_HTML) { - switch (c) { - MATCH('"', """); - MATCH('\'', "'"); - MATCH('<', "<"); - MATCH('>', ">"); - MATCH('&', "&"); - } - //for other characters, we fall through to the non-HTML-escaped part + return true + + inline bool TBuf::EscapedWriteChar(const char* beg, const char* cur, EHtmlEscapeMode hem) { + unsigned char c = *cur; + if (hem == HEM_ESCAPE_HTML) { + switch (c) { + MATCH('"', """); + MATCH('\'', "'"); + MATCH('<', "<"); + MATCH('>', ">"); + MATCH('&', "&"); + } + //for other characters, we fall through to the non-HTML-escaped part } - if (hem == HEM_RELAXED && c == '/') - return false; - - if (hem != HEM_UNSAFE) { - switch (c) { - case '/': - UnsafeWriteRawBytes(beg, cur - beg); - UnsafeWriteRawBytes("\\/", 2); - return true; - case '<': - case '>': - case '\'': - UnsafeWriteRawBytes(beg, cur - beg); - WriteHexEscape(c); - return true; - } - // for other characters, fall through to the non-escaped part + if (hem == HEM_RELAXED && c == '/') + return false; + + if (hem != HEM_UNSAFE) { + switch (c) { + case '/': + UnsafeWriteRawBytes(beg, cur - beg); + UnsafeWriteRawBytes("\\/", 2); + return true; + case '<': + case '>': + case '\'': + UnsafeWriteRawBytes(beg, cur - beg); + WriteHexEscape(c); + return true; + } + // for other characters, fall through to the non-escaped part } - switch (c) { + switch (c) { MATCH('"', "\\\""); MATCH('\\', "\\\\"); MATCH('\b', "\\b"); @@ -366,152 +366,152 @@ namespace NJsonWriter { MATCH('\n', "\\n"); MATCH('\r', "\\r"); MATCH('\t', "\\t"); - } - if (c < 0x20) { - UnsafeWriteRawBytes(beg, cur - beg); - WriteHexEscape(c); - return true; - } - - return false; + } + if (c < 0x20) { + UnsafeWriteRawBytes(beg, cur - beg); + WriteHexEscape(c); + return true; + } + + return false; } #undef MATCH - static bool LessStrPtr(const TString* a, const TString* b) { - return *a < *b; - } + static bool LessStrPtr(const TString* a, const TString* b) { + return *a < *b; + } TValueContext TBuf::WriteJsonValue(const NJson::TJsonValue* v, bool sortKeys, EFloatToStringMode mode, int ndigits) { - using namespace NJson; - switch (v->GetType()) { - default: - case JSON_NULL: - WriteNull(); - break; - case JSON_BOOLEAN: - WriteBool(v->GetBoolean()); - break; - case JSON_DOUBLE: + using namespace NJson; + switch (v->GetType()) { + default: + case JSON_NULL: + WriteNull(); + break; + case JSON_BOOLEAN: + WriteBool(v->GetBoolean()); + break; + case JSON_DOUBLE: WriteDouble(v->GetDouble(), mode, ndigits); - break; - case JSON_INTEGER: - WriteLongLong(v->GetInteger()); - break; - case JSON_UINTEGER: - WriteULongLong(v->GetUInteger()); - break; - case JSON_STRING: - WriteString(v->GetString()); - break; - case JSON_ARRAY: { - BeginList(); - const TJsonValue::TArray& arr = v->GetArray(); - for (const auto& it : arr) + break; + case JSON_INTEGER: + WriteLongLong(v->GetInteger()); + break; + case JSON_UINTEGER: + WriteULongLong(v->GetUInteger()); + break; + case JSON_STRING: + WriteString(v->GetString()); + break; + case JSON_ARRAY: { + BeginList(); + const TJsonValue::TArray& arr = v->GetArray(); + for (const auto& it : arr) WriteJsonValue(&it, sortKeys, mode, ndigits); - EndList(); - break; + EndList(); + break; } - case JSON_MAP: { - BeginObject(); - const TJsonValue::TMapType& map = v->GetMap(); - if (sortKeys) { - const size_t oldsz = Keys.size(); - Keys.reserve(map.size() + oldsz); - for (const auto& it : map) { - Keys.push_back(&(it.first)); - } - Sort(Keys.begin() + oldsz, Keys.end(), LessStrPtr); - for (size_t i = oldsz, sz = Keys.size(); i < sz; ++i) { - TJsonValue::TMapType::const_iterator kv = map.find(*Keys[i]); - WriteKey(kv->first); + case JSON_MAP: { + BeginObject(); + const TJsonValue::TMapType& map = v->GetMap(); + if (sortKeys) { + const size_t oldsz = Keys.size(); + Keys.reserve(map.size() + oldsz); + for (const auto& it : map) { + Keys.push_back(&(it.first)); + } + Sort(Keys.begin() + oldsz, Keys.end(), LessStrPtr); + for (size_t i = oldsz, sz = Keys.size(); i < sz; ++i) { + TJsonValue::TMapType::const_iterator kv = map.find(*Keys[i]); + WriteKey(kv->first); WriteJsonValue(&kv->second, sortKeys, mode, ndigits); - } - Keys.resize(oldsz); - } else { - for (const auto& it : map) { - WriteKey(it.first); + } + Keys.resize(oldsz); + } else { + for (const auto& it : map) { + WriteKey(it.first); WriteJsonValue(&it.second, sortKeys, mode, ndigits); - } - } - EndObject(); - break; + } + } + EndObject(); + break; } } - return TValueContext(*this); + return TValueContext(*this); } - - TPairContext TBuf::UnsafeWritePair(const TStringBuf& s) { - if (Y_UNLIKELY(StackTop() != JE_OBJECT)) { - ythrow TError() << "JSON writer: key:value pair written outside of an object"; - } - WriteComma(); - UnsafeWriteRawBytes(s); - return TPairContext(*this); + + TPairContext TBuf::UnsafeWritePair(const TStringBuf& s) { + if (Y_UNLIKELY(StackTop() != JE_OBJECT)) { + ythrow TError() << "JSON writer: key:value pair written outside of an object"; + } + WriteComma(); + UnsafeWriteRawBytes(s); + return TPairContext(*this); } - void TBuf::UnsafeWriteValue(const TStringBuf& s) { - BeginValue(); - UnsafeWriteRawBytes(s); - EndValue(); + void TBuf::UnsafeWriteValue(const TStringBuf& s) { + BeginValue(); + UnsafeWriteRawBytes(s); + EndValue(); } - void TBuf::UnsafeWriteValue(const char* s, size_t len) { - BeginValue(); - UnsafeWriteRawBytes(s, len); - EndValue(); - } + void TBuf::UnsafeWriteValue(const char* s, size_t len) { + BeginValue(); + UnsafeWriteRawBytes(s, len); + EndValue(); + } - void TBuf::UnsafeWriteRawBytes(const char* src, size_t len) { - Stream->Write(src, len); - } + void TBuf::UnsafeWriteRawBytes(const char* src, size_t len) { + Stream->Write(src, len); + } - void TBuf::UnsafeWriteRawBytes(const TStringBuf& s) { + void TBuf::UnsafeWriteRawBytes(const TStringBuf& s) { UnsafeWriteRawBytes(s.data(), s.size()); - } - - const TString& TBuf::Str() const { - if (!StringStream) { - ythrow TError() << "JSON writer: Str() called " - "but writing to an external stream"; - } - if (!(Stack.size() == 1 && StackTop() == JE_OUTER_SPACE)) { - ythrow TError() << "JSON writer: incomplete object converted to string"; - } - return StringStream->Str(); - } - - void TBuf::FlushTo(IOutputStream* stream) { - if (!StringStream) { - ythrow TError() << "JSON writer: FlushTo() called " - "but writing to an external stream"; - } - stream->Write(StringStream->Str()); - StringStream->Clear(); - } - - TString WrapJsonToCallback(const TBuf& buf, TStringBuf callback) { - if (!callback) { - return buf.Str(); - } else { - return TString::Join(callback, "(", buf.Str(), ")"); - } - } - - TBufState TBuf::State() const { - return TBufState{NeedComma, NeedNewline, Stack}; - } - - void TBuf::Reset(const TBufState& from) { - NeedComma = from.NeedComma; - NeedNewline = from.NeedNewline; - Stack = from.Stack; - } - - void TBuf::Reset(TBufState&& from) { - NeedComma = from.NeedComma; - NeedNewline = from.NeedNewline; - Stack.swap(from.Stack); - } + } + + const TString& TBuf::Str() const { + if (!StringStream) { + ythrow TError() << "JSON writer: Str() called " + "but writing to an external stream"; + } + if (!(Stack.size() == 1 && StackTop() == JE_OUTER_SPACE)) { + ythrow TError() << "JSON writer: incomplete object converted to string"; + } + return StringStream->Str(); + } + + void TBuf::FlushTo(IOutputStream* stream) { + if (!StringStream) { + ythrow TError() << "JSON writer: FlushTo() called " + "but writing to an external stream"; + } + stream->Write(StringStream->Str()); + StringStream->Clear(); + } + + TString WrapJsonToCallback(const TBuf& buf, TStringBuf callback) { + if (!callback) { + return buf.Str(); + } else { + return TString::Join(callback, "(", buf.Str(), ")"); + } + } + + TBufState TBuf::State() const { + return TBufState{NeedComma, NeedNewline, Stack}; + } + + void TBuf::Reset(const TBufState& from) { + NeedComma = from.NeedComma; + NeedNewline = from.NeedNewline; + Stack = from.Stack; + } + + void TBuf::Reset(TBufState&& from) { + NeedComma = from.NeedComma; + NeedNewline = from.NeedNewline; + Stack.swap(from.Stack); + } } diff --git a/library/cpp/json/writer/json.h b/library/cpp/json/writer/json.h index 0aae2531b9..eca04a510a 100644 --- a/library/cpp/json/writer/json.h +++ b/library/cpp/json/writer/json.h @@ -39,7 +39,7 @@ namespace NJsonWriter { TVector<EJsonEntity> Stack; }; - class TBuf : TNonCopyable { + class TBuf : TNonCopyable { public: TBuf(EHtmlEscapeMode mode = HEM_DONT_ESCAPE_HTML, IOutputStream* stream = nullptr); @@ -183,8 +183,8 @@ namespace NJsonWriter { protected: TValueWriter(TBuf& buf) : Buf(buf) - { - } + { + } friend class TBuf; protected: @@ -196,25 +196,25 @@ namespace NJsonWriter { TBuf& EndList() { return Buf.EndList(); } - TString Str() const { - return Buf.Str(); - } - + TString Str() const { + return Buf.Str(); + } + private: TValueContext(TBuf& buf) : TValueWriter<TValueContext>(buf) - { - } + { + } friend class TBuf; friend class TValueWriter<TValueContext>; }; class TAfterColonContext: public TValueWriter<TPairContext> { private: - TAfterColonContext(TBuf& iBuf) - : TValueWriter<TPairContext>(iBuf) - { - } + TAfterColonContext(TBuf& iBuf) + : TValueWriter<TPairContext>(iBuf) + { + } friend class TBuf; friend class TPairContext; }; @@ -275,15 +275,15 @@ namespace NJsonWriter { JSON_VALUE_WRITER_WRAP(UnsafeWriteValue, (const TStringBuf& arg), (arg)) #undef JSON_VALUE_WRITER_WRAP - template <typename TOutContext> - TValueContext TValueWriter<TOutContext>::BeginList() { + template <typename TOutContext> + TValueContext TValueWriter<TOutContext>::BeginList() { return Buf.BeginList(); } - template <typename TOutContext> - TPairContext TValueWriter<TOutContext>::BeginObject() { + template <typename TOutContext> + TPairContext TValueWriter<TOutContext>::BeginObject() { return Buf.BeginObject(); } - TString WrapJsonToCallback(const TBuf& buf, TStringBuf callback); + TString WrapJsonToCallback(const TBuf& buf, TStringBuf callback); } diff --git a/library/cpp/json/writer/json_ut.cpp b/library/cpp/json/writer/json_ut.cpp index 9980555683..e8d0be3566 100644 --- a/library/cpp/json/writer/json_ut.cpp +++ b/library/cpp/json/writer/json_ut.cpp @@ -16,12 +16,12 @@ Y_UNIT_TEST_SUITE(JsonWriter) { .UnsafeWritePair("\"xk\":13") .WriteKey("key2") .BeginList() - .BeginObject() - .EndObject() - .BeginObject() - .EndObject() + .BeginObject() + .EndObject() + .BeginObject() + .EndObject() .EndList() - .EndObject(); + .EndObject(); w.WriteInt(43); w.UnsafeWriteValue("\"x\""); w.WriteString("..."); @@ -75,11 +75,11 @@ Y_UNIT_TEST_SUITE(JsonWriter) { Y_UNIT_TEST(BareKey) { NJsonWriter::TBuf w; w.BeginObject() - .CompatWriteKeyWithoutQuotes("p") - .WriteInt(1) - .CompatWriteKeyWithoutQuotes("n") - .WriteInt(0) - .EndObject(); + .CompatWriteKeyWithoutQuotes("p") + .WriteInt(1) + .CompatWriteKeyWithoutQuotes("n") + .WriteInt(0) + .EndObject(); TString ws = w.Str(); const char* exp = "{p:1,n:0}"; UNIT_ASSERT_STRINGS_EQUAL(ws.c_str(), exp); @@ -108,12 +108,12 @@ Y_UNIT_TEST_SUITE(JsonWriter) { .WriteInt(1) .WriteString("hello") .BeginObject() - .WriteKey("abc") - .WriteInt(3) - .WriteKey("def") - .WriteInt(4) + .WriteKey("abc") + .WriteInt(3) + .WriteKey("def") + .WriteInt(4) .EndObject() - .EndList(); + .EndList(); const char* exp = "[\n" " 1,\n" " \"hello\",\n" @@ -196,12 +196,12 @@ Y_UNIT_TEST_SUITE(JsonWriter) { .WriteFloat(244.13854, PREC_NDIGITS, 4) .WriteFloat(10385.8324, PREC_POINT_DIGITS, 2) .BeginObject() - .WriteKey("1") - .WriteDouble(1111.71, PREC_POINT_DIGITS, 0) - .WriteKey("2") - .WriteDouble(1111.71, PREC_NDIGITS, 1) + .WriteKey("1") + .WriteDouble(1111.71, PREC_POINT_DIGITS, 0) + .WriteKey("2") + .WriteDouble(1111.71, PREC_NDIGITS, 1) .EndObject() - .EndList(); + .EndList(); const char exp[] = "[0.123457,0.1234567899,0.316,244.1,10385.83,{\"1\":1112,\"2\":1e+03}]"; UNIT_ASSERT_STRINGS_EQUAL(exp, buf.Str()); } @@ -238,17 +238,17 @@ Y_UNIT_TEST_SUITE(JsonWriter) { buf.SetWriteNanAsString(); buf.BeginObject() - .WriteKey("nanvalue") - .WriteFloat(std::numeric_limits<double>::quiet_NaN()) - .WriteKey("infvalue") - .WriteFloat(std::numeric_limits<double>::infinity()) - .WriteKey("minus_infvalue") - .WriteFloat(-std::numeric_limits<float>::infinity()) + .WriteKey("nanvalue") + .WriteFloat(std::numeric_limits<double>::quiet_NaN()) + .WriteKey("infvalue") + .WriteFloat(std::numeric_limits<double>::infinity()) + .WriteKey("minus_infvalue") + .WriteFloat(-std::numeric_limits<float>::infinity()) .WriteKey("l") - .BeginList() - .WriteFloat(std::numeric_limits<float>::quiet_NaN()) - .EndList() - .EndObject(); + .BeginList() + .WriteFloat(std::numeric_limits<float>::quiet_NaN()) + .EndList() + .EndObject(); UNIT_ASSERT_STRINGS_EQUAL(buf.Str(), R"raw_json({"nanvalue":"nan","infvalue":"inf","minus_infvalue":"-inf","l":["nan"]})raw_json"); } @@ -256,11 +256,11 @@ Y_UNIT_TEST_SUITE(JsonWriter) { { NJsonWriter::TBuf buf; buf.BeginObject() - .WriteKey("<>&") - .WriteString("Ololo") - .UnsafeWriteKey("<>&") - .WriteString("Ololo2") - .EndObject(); + .WriteKey("<>&") + .WriteString("Ololo") + .UnsafeWriteKey("<>&") + .WriteString("Ololo2") + .EndObject(); UNIT_ASSERT_STRINGS_EQUAL(buf.Str(), R"({"\u003C\u003E&":"Ololo","<>&":"Ololo2"})"); } diff --git a/library/cpp/json/writer/json_value.cpp b/library/cpp/json/writer/json_value.cpp index c61e8d1dc4..4e85a6bc08 100644 --- a/library/cpp/json/writer/json_value.cpp +++ b/library/cpp/json/writer/json_value.cpp @@ -1,12 +1,12 @@ #include "json_value.h" #include "json.h" -#include <util/generic/ymath.h> +#include <util/generic/ymath.h> #include <util/generic/ylimits.h> #include <util/generic/utility.h> -#include <util/generic/singleton.h> +#include <util/generic/singleton.h> #include <util/stream/str.h> -#include <util/stream/output.h> +#include <util/stream/output.h> #include <util/string/cast.h> #include <util/string/type.h> #include <util/string/vector.h> @@ -59,7 +59,7 @@ AreJsonArraysEqual(const NJson::TJsonValue& lhs, const NJson::TJsonValue& rhs) { return false; for (TArray::const_iterator lhsIt = lhsArray.begin(), rhsIt = rhsArray.begin(); - lhsIt != lhsArray.end(); ++lhsIt, ++rhsIt) { + lhsIt != lhsArray.end(); ++lhsIt, ++rhsIt) { if (*lhsIt != *rhsIt) return false; } @@ -68,926 +68,926 @@ AreJsonArraysEqual(const NJson::TJsonValue& lhs, const NJson::TJsonValue& rhs) { } namespace NJson { - const TJsonValue TJsonValue::UNDEFINED{}; - - TJsonValue::TJsonValue(const EJsonValueType type) { - SetType(type); - } - - TJsonValue::TJsonValue(TJsonValue&& vval) noexcept - : Type(JSON_UNDEFINED) - { - vval.SwapWithUndefined(*this); - Zero(vval.Value); - } - - TJsonValue::TJsonValue(const TJsonValue& val) - : Type(val.Type) - { - switch (Type) { - case JSON_STRING: - new (&Value.String) TString(val.GetString()); - break; - case JSON_MAP: - Value.Map = new TMapType(val.GetMap()); - break; - case JSON_ARRAY: - Value.Array = new TArray(val.GetArray()); - break; - case JSON_UNDEFINED: - case JSON_NULL: - case JSON_BOOLEAN: - case JSON_INTEGER: - case JSON_UINTEGER: - case JSON_DOUBLE: - std::memcpy(&Value, &val.Value, sizeof(Value)); - break; - } - } - - TJsonValue& TJsonValue::operator=(const TJsonValue& val) { - if (this == &val) - return *this; - TJsonValue tmp(val); - tmp.Swap(*this); + const TJsonValue TJsonValue::UNDEFINED{}; + + TJsonValue::TJsonValue(const EJsonValueType type) { + SetType(type); + } + + TJsonValue::TJsonValue(TJsonValue&& vval) noexcept + : Type(JSON_UNDEFINED) + { + vval.SwapWithUndefined(*this); + Zero(vval.Value); + } + + TJsonValue::TJsonValue(const TJsonValue& val) + : Type(val.Type) + { + switch (Type) { + case JSON_STRING: + new (&Value.String) TString(val.GetString()); + break; + case JSON_MAP: + Value.Map = new TMapType(val.GetMap()); + break; + case JSON_ARRAY: + Value.Array = new TArray(val.GetArray()); + break; + case JSON_UNDEFINED: + case JSON_NULL: + case JSON_BOOLEAN: + case JSON_INTEGER: + case JSON_UINTEGER: + case JSON_DOUBLE: + std::memcpy(&Value, &val.Value, sizeof(Value)); + break; + } + } + + TJsonValue& TJsonValue::operator=(const TJsonValue& val) { + if (this == &val) + return *this; + TJsonValue tmp(val); + tmp.Swap(*this); return *this; - } + } - TJsonValue& TJsonValue::operator=(TJsonValue&& val) noexcept { - if (this == &val) - return *this; - TJsonValue tmp(std::move(val)); - tmp.Swap(*this); + TJsonValue& TJsonValue::operator=(TJsonValue&& val) noexcept { + if (this == &val) + return *this; + TJsonValue tmp(std::move(val)); + tmp.Swap(*this); return *this; - } - - TJsonValue::TJsonValue(const bool value) noexcept { - SetType(JSON_BOOLEAN); - Value.Boolean = value; - } - - TJsonValue::TJsonValue(const long long value) noexcept { - SetType(JSON_INTEGER); - Value.Integer = value; - } - - TJsonValue::TJsonValue(const unsigned long long value) noexcept { - SetType(JSON_UINTEGER); - Value.UInteger = value; - } - - TJsonValue::TJsonValue(const int value) noexcept { - SetType(JSON_INTEGER); - Value.Integer = value; - } - - TJsonValue::TJsonValue(const unsigned int value) noexcept { - SetType(JSON_UINTEGER); - Value.UInteger = value; - } - - TJsonValue::TJsonValue(const long value) noexcept { - SetType(JSON_INTEGER); - Value.Integer = value; - } - - TJsonValue::TJsonValue(const unsigned long value) noexcept { - SetType(JSON_UINTEGER); - Value.UInteger = value; - } - - TJsonValue::TJsonValue(const double value) noexcept { - SetType(JSON_DOUBLE); - Value.Double = value; - } + } + + TJsonValue::TJsonValue(const bool value) noexcept { + SetType(JSON_BOOLEAN); + Value.Boolean = value; + } + + TJsonValue::TJsonValue(const long long value) noexcept { + SetType(JSON_INTEGER); + Value.Integer = value; + } + + TJsonValue::TJsonValue(const unsigned long long value) noexcept { + SetType(JSON_UINTEGER); + Value.UInteger = value; + } + + TJsonValue::TJsonValue(const int value) noexcept { + SetType(JSON_INTEGER); + Value.Integer = value; + } + + TJsonValue::TJsonValue(const unsigned int value) noexcept { + SetType(JSON_UINTEGER); + Value.UInteger = value; + } + + TJsonValue::TJsonValue(const long value) noexcept { + SetType(JSON_INTEGER); + Value.Integer = value; + } + + TJsonValue::TJsonValue(const unsigned long value) noexcept { + SetType(JSON_UINTEGER); + Value.UInteger = value; + } + + TJsonValue::TJsonValue(const double value) noexcept { + SetType(JSON_DOUBLE); + Value.Double = value; + } TJsonValue::TJsonValue(TString value) { - SetType(JSON_STRING); - Value.String = std::move(value); - } - - TJsonValue::TJsonValue(const TStringBuf value) { - SetType(JSON_STRING); - Value.String = value; - } - - TJsonValue::TJsonValue(const char* value) { - SetType(JSON_STRING); - Value.String = value; - } - - EJsonValueType TJsonValue::GetType() const noexcept { - return Type; - } - - TJsonValue& TJsonValue::SetType(const EJsonValueType type) { - if (Type == type) - return *this; - - Clear(); - Type = type; - - switch (Type) { - case JSON_STRING: - new (&Value.String) TString(); - break; - case JSON_MAP: - Value.Map = new TMapType(); - break; - case JSON_ARRAY: - Value.Array = new TArray(); - break; - case JSON_UNDEFINED: - case JSON_NULL: - case JSON_BOOLEAN: - case JSON_INTEGER: - case JSON_UINTEGER: - case JSON_DOUBLE: - break; - } - - return *this; - } - - TJsonValue& TJsonValue::SetValue(const TJsonValue& value) { - return *this = value; - } - - TJsonValue& TJsonValue::SetValue(TJsonValue&& value) { - *this = std::move(value); - return *this; - } - - TJsonValue& TJsonValue::InsertValue(const TString& key, const TJsonValue& value) { - SetType(JSON_MAP); - return (*Value.Map)[key] = value; - } - - TJsonValue& TJsonValue::InsertValue(const TStringBuf key, const TJsonValue& value) { - SetType(JSON_MAP); - return (*Value.Map)[key] = value; - } - - TJsonValue& TJsonValue::InsertValue(const char* key, const TJsonValue& value) { - SetType(JSON_MAP); - return (*Value.Map)[key] = value; - } - - TJsonValue& TJsonValue::InsertValue(const TString& key, TJsonValue&& value) { - SetType(JSON_MAP); - return (*Value.Map)[key] = std::move(value); - } - - TJsonValue& TJsonValue::InsertValue(const TStringBuf key, TJsonValue&& value) { - SetType(JSON_MAP); - return (*Value.Map)[key] = std::move(value); - } - - TJsonValue& TJsonValue::InsertValue(const char* key, TJsonValue&& value) { - SetType(JSON_MAP); - return (*Value.Map)[key] = std::move(value); - } - - TJsonValue& TJsonValue::Back() { + SetType(JSON_STRING); + Value.String = std::move(value); + } + + TJsonValue::TJsonValue(const TStringBuf value) { + SetType(JSON_STRING); + Value.String = value; + } + + TJsonValue::TJsonValue(const char* value) { + SetType(JSON_STRING); + Value.String = value; + } + + EJsonValueType TJsonValue::GetType() const noexcept { + return Type; + } + + TJsonValue& TJsonValue::SetType(const EJsonValueType type) { + if (Type == type) + return *this; + + Clear(); + Type = type; + + switch (Type) { + case JSON_STRING: + new (&Value.String) TString(); + break; + case JSON_MAP: + Value.Map = new TMapType(); + break; + case JSON_ARRAY: + Value.Array = new TArray(); + break; + case JSON_UNDEFINED: + case JSON_NULL: + case JSON_BOOLEAN: + case JSON_INTEGER: + case JSON_UINTEGER: + case JSON_DOUBLE: + break; + } + + return *this; + } + + TJsonValue& TJsonValue::SetValue(const TJsonValue& value) { + return *this = value; + } + + TJsonValue& TJsonValue::SetValue(TJsonValue&& value) { + *this = std::move(value); + return *this; + } + + TJsonValue& TJsonValue::InsertValue(const TString& key, const TJsonValue& value) { + SetType(JSON_MAP); + return (*Value.Map)[key] = value; + } + + TJsonValue& TJsonValue::InsertValue(const TStringBuf key, const TJsonValue& value) { + SetType(JSON_MAP); + return (*Value.Map)[key] = value; + } + + TJsonValue& TJsonValue::InsertValue(const char* key, const TJsonValue& value) { + SetType(JSON_MAP); + return (*Value.Map)[key] = value; + } + + TJsonValue& TJsonValue::InsertValue(const TString& key, TJsonValue&& value) { + SetType(JSON_MAP); + return (*Value.Map)[key] = std::move(value); + } + + TJsonValue& TJsonValue::InsertValue(const TStringBuf key, TJsonValue&& value) { + SetType(JSON_MAP); + return (*Value.Map)[key] = std::move(value); + } + + TJsonValue& TJsonValue::InsertValue(const char* key, TJsonValue&& value) { + SetType(JSON_MAP); + return (*Value.Map)[key] = std::move(value); + } + + TJsonValue& TJsonValue::Back() { BackChecks(); return Value.Array->back(); } const TJsonValue& TJsonValue::Back() const { BackChecks(); - return Value.Array->back(); - } - - TJsonValue& TJsonValue::AppendValue(const TJsonValue& value) { - SetType(JSON_ARRAY); - Value.Array->push_back(value); - return Value.Array->back(); - } - - TJsonValue& TJsonValue::AppendValue(TJsonValue&& value) { - SetType(JSON_ARRAY); - Value.Array->push_back(std::move(value)); - return Value.Array->back(); - } - - void TJsonValue::EraseValue(const TStringBuf key) { - if (IsMap()) { - TMapType::iterator it = Value.Map->find(key); - if (it != Value.Map->end()) - Value.Map->erase(it); + return Value.Array->back(); + } + + TJsonValue& TJsonValue::AppendValue(const TJsonValue& value) { + SetType(JSON_ARRAY); + Value.Array->push_back(value); + return Value.Array->back(); + } + + TJsonValue& TJsonValue::AppendValue(TJsonValue&& value) { + SetType(JSON_ARRAY); + Value.Array->push_back(std::move(value)); + return Value.Array->back(); + } + + void TJsonValue::EraseValue(const TStringBuf key) { + if (IsMap()) { + TMapType::iterator it = Value.Map->find(key); + if (it != Value.Map->end()) + Value.Map->erase(it); + } + } + + void TJsonValue::EraseValue(const size_t index) { + if (IsArray()) { + if (index >= Value.Array->size()) { + return; + } + TArray::iterator it = Value.Array->begin() + index; + Value.Array->erase(it); + } + } + + void TJsonValue::Clear() noexcept { + switch (Type) { + case JSON_STRING: + Value.String.~TString(); + break; + case JSON_MAP: + delete Value.Map; + break; + case JSON_ARRAY: + delete Value.Array; + break; + case JSON_UNDEFINED: + case JSON_NULL: + case JSON_BOOLEAN: + case JSON_INTEGER: + case JSON_UINTEGER: + case JSON_DOUBLE: + break; } + Zero(Value); + Type = JSON_UNDEFINED; } - void TJsonValue::EraseValue(const size_t index) { - if (IsArray()) { - if (index >= Value.Array->size()) { - return; - } - TArray::iterator it = Value.Array->begin() + index; - Value.Array->erase(it); - } - } + TJsonValue& TJsonValue::operator[](const size_t idx) { + SetType(JSON_ARRAY); + if (Value.Array->size() <= idx) + Value.Array->resize(idx + 1); + return (*Value.Array)[idx]; + } - void TJsonValue::Clear() noexcept { - switch (Type) { - case JSON_STRING: - Value.String.~TString(); - break; - case JSON_MAP: - delete Value.Map; - break; - case JSON_ARRAY: - delete Value.Array; - break; - case JSON_UNDEFINED: - case JSON_NULL: - case JSON_BOOLEAN: - case JSON_INTEGER: - case JSON_UINTEGER: - case JSON_DOUBLE: - break; - } - Zero(Value); - Type = JSON_UNDEFINED; + TJsonValue& TJsonValue::operator[](const TStringBuf& key) { + SetType(JSON_MAP); + return (*Value.Map)[key]; } - TJsonValue& TJsonValue::operator[](const size_t idx) { - SetType(JSON_ARRAY); - if (Value.Array->size() <= idx) - Value.Array->resize(idx + 1); - return (*Value.Array)[idx]; - } + namespace { + struct TDefaultsHolder { + const TString String{}; + const TJsonValue::TMapType Map{}; + const TJsonValue::TArray Array{}; + const TJsonValue Value{}; + }; + } - TJsonValue& TJsonValue::operator[](const TStringBuf& key) { - SetType(JSON_MAP); - return (*Value.Map)[key]; - } + const TJsonValue& TJsonValue::operator[](const size_t idx) const noexcept { + const TJsonValue* ret = nullptr; + if (GetValuePointer(idx, &ret)) + return *ret; - namespace { - struct TDefaultsHolder { - const TString String{}; - const TJsonValue::TMapType Map{}; - const TJsonValue::TArray Array{}; - const TJsonValue Value{}; - }; - } + return Singleton<TDefaultsHolder>()->Value; + } - const TJsonValue& TJsonValue::operator[](const size_t idx) const noexcept { - const TJsonValue* ret = nullptr; - if (GetValuePointer(idx, &ret)) - return *ret; + const TJsonValue& TJsonValue::operator[](const TStringBuf& key) const noexcept { + const TJsonValue* ret = nullptr; + if (GetValuePointer(key, &ret)) + return *ret; - return Singleton<TDefaultsHolder>()->Value; - } + return Singleton<TDefaultsHolder>()->Value; + } - const TJsonValue& TJsonValue::operator[](const TStringBuf& key) const noexcept { - const TJsonValue* ret = nullptr; - if (GetValuePointer(key, &ret)) - return *ret; + bool TJsonValue::GetBoolean() const { + return Type != JSON_BOOLEAN ? false : Value.Boolean; + } - return Singleton<TDefaultsHolder>()->Value; - } + long long TJsonValue::GetInteger() const { + if (!IsInteger()) + return 0; - bool TJsonValue::GetBoolean() const { - return Type != JSON_BOOLEAN ? false : Value.Boolean; - } + switch (Type) { + case JSON_INTEGER: + return Value.Integer; - long long TJsonValue::GetInteger() const { - if (!IsInteger()) - return 0; + case JSON_UINTEGER: + return Value.UInteger; - switch (Type) { - case JSON_INTEGER: - return Value.Integer; - - case JSON_UINTEGER: - return Value.UInteger; - - case JSON_DOUBLE: - return Value.Double; + case JSON_DOUBLE: + return Value.Double; - default: - Y_ASSERT(false && "Unexpected type."); - return 0; - } + default: + Y_ASSERT(false && "Unexpected type."); + return 0; + } } - unsigned long long TJsonValue::GetUInteger() const { - if (!IsUInteger()) - return 0; + unsigned long long TJsonValue::GetUInteger() const { + if (!IsUInteger()) + return 0; - switch (Type) { - case JSON_UINTEGER: - return Value.UInteger; + switch (Type) { + case JSON_UINTEGER: + return Value.UInteger; - case JSON_INTEGER: - return Value.Integer; + case JSON_INTEGER: + return Value.Integer; - case JSON_DOUBLE: - return Value.Double; + case JSON_DOUBLE: + return Value.Double; - default: - Y_ASSERT(false && "Unexpected type."); - return 0; - } + default: + Y_ASSERT(false && "Unexpected type."); + return 0; + } } - double TJsonValue::GetDouble() const { - if (!IsDouble()) - return 0.0; + double TJsonValue::GetDouble() const { + if (!IsDouble()) + return 0.0; - switch (Type) { - case JSON_DOUBLE: - return Value.Double; + switch (Type) { + case JSON_DOUBLE: + return Value.Double; - case JSON_INTEGER: - return Value.Integer; + case JSON_INTEGER: + return Value.Integer; - case JSON_UINTEGER: - return Value.UInteger; + case JSON_UINTEGER: + return Value.UInteger; - default: - Y_ASSERT(false && "Unexpected type."); - return 0.0; - } + default: + Y_ASSERT(false && "Unexpected type."); + return 0.0; + } } - const TString& TJsonValue::GetString() const { - return Type != JSON_STRING ? Singleton<TDefaultsHolder>()->String : Value.String; - } + const TString& TJsonValue::GetString() const { + return Type != JSON_STRING ? Singleton<TDefaultsHolder>()->String : Value.String; + } - const TJsonValue::TMapType& TJsonValue::GetMap() const { - return Type != JSON_MAP ? Singleton<TDefaultsHolder>()->Map : *Value.Map; - } + const TJsonValue::TMapType& TJsonValue::GetMap() const { + return Type != JSON_MAP ? Singleton<TDefaultsHolder>()->Map : *Value.Map; + } - const TJsonValue::TArray& TJsonValue::GetArray() const { - return (Type != JSON_ARRAY) ? Singleton<TDefaultsHolder>()->Array : *Value.Array; - } + const TJsonValue::TArray& TJsonValue::GetArray() const { + return (Type != JSON_ARRAY) ? Singleton<TDefaultsHolder>()->Array : *Value.Array; + } - bool TJsonValue::GetBooleanSafe() const { - if (Type != JSON_BOOLEAN) - ythrow TJsonException() << "Not a boolean"; + bool TJsonValue::GetBooleanSafe() const { + if (Type != JSON_BOOLEAN) + ythrow TJsonException() << "Not a boolean"; - return Value.Boolean; - } + return Value.Boolean; + } - long long TJsonValue::GetIntegerSafe() const { - if (!IsInteger()) - ythrow TJsonException() << "Not an integer"; + long long TJsonValue::GetIntegerSafe() const { + if (!IsInteger()) + ythrow TJsonException() << "Not an integer"; - return GetInteger(); - } + return GetInteger(); + } - unsigned long long TJsonValue::GetUIntegerSafe() const { - if (!IsUInteger()) - ythrow TJsonException() << "Not an unsigned integer"; + unsigned long long TJsonValue::GetUIntegerSafe() const { + if (!IsUInteger()) + ythrow TJsonException() << "Not an unsigned integer"; - return GetUInteger(); - } + return GetUInteger(); + } - double TJsonValue::GetDoubleSafe() const { - if (!IsDouble()) - ythrow TJsonException() << "Not a double"; + double TJsonValue::GetDoubleSafe() const { + if (!IsDouble()) + ythrow TJsonException() << "Not a double"; - return GetDouble(); - } + return GetDouble(); + } - const TString& TJsonValue::GetStringSafe() const { - if (Type != JSON_STRING) - ythrow TJsonException() << "Not a string"; + const TString& TJsonValue::GetStringSafe() const { + if (Type != JSON_STRING) + ythrow TJsonException() << "Not a string"; - return Value.String; - } + return Value.String; + } - bool TJsonValue::GetBooleanSafe(const bool defaultValue) const { - if (Type == JSON_UNDEFINED) - return defaultValue; + bool TJsonValue::GetBooleanSafe(const bool defaultValue) const { + if (Type == JSON_UNDEFINED) + return defaultValue; - return GetBooleanSafe(); - } + return GetBooleanSafe(); + } - long long TJsonValue::GetIntegerSafe(const long long defaultValue) const { - if (Type == JSON_UNDEFINED) - return defaultValue; + long long TJsonValue::GetIntegerSafe(const long long defaultValue) const { + if (Type == JSON_UNDEFINED) + return defaultValue; - return GetIntegerSafe(); - } + return GetIntegerSafe(); + } - unsigned long long TJsonValue::GetUIntegerSafe(const unsigned long long defaultValue) const { - if (Type == JSON_UNDEFINED) - return defaultValue; + unsigned long long TJsonValue::GetUIntegerSafe(const unsigned long long defaultValue) const { + if (Type == JSON_UNDEFINED) + return defaultValue; - return GetUIntegerSafe(); - } + return GetUIntegerSafe(); + } - double TJsonValue::GetDoubleSafe(const double defaultValue) const { - if (Type == JSON_UNDEFINED) - return defaultValue; + double TJsonValue::GetDoubleSafe(const double defaultValue) const { + if (Type == JSON_UNDEFINED) + return defaultValue; - return GetDoubleSafe(); - } + return GetDoubleSafe(); + } - TString TJsonValue::GetStringSafe(const TString& defaultValue) const { - if (Type == JSON_UNDEFINED) - return defaultValue; + TString TJsonValue::GetStringSafe(const TString& defaultValue) const { + if (Type == JSON_UNDEFINED) + return defaultValue; - return GetStringSafe(); - } + return GetStringSafe(); + } - const TJsonValue::TMapType& TJsonValue::GetMapSafe() const { - if (Type != JSON_MAP) - ythrow TJsonException() << "Not a map"; + const TJsonValue::TMapType& TJsonValue::GetMapSafe() const { + if (Type != JSON_MAP) + ythrow TJsonException() << "Not a map"; - return *Value.Map; - } + return *Value.Map; + } - TJsonValue::TMapType& TJsonValue::GetMapSafe() { - return const_cast<TJsonValue::TMapType&>(const_cast<const TJsonValue*>(this)->GetMapSafe()); - } + TJsonValue::TMapType& TJsonValue::GetMapSafe() { + return const_cast<TJsonValue::TMapType&>(const_cast<const TJsonValue*>(this)->GetMapSafe()); + } - const TJsonValue::TArray& TJsonValue::GetArraySafe() const { - if (Type != JSON_ARRAY) - ythrow TJsonException() << "Not an array"; + const TJsonValue::TArray& TJsonValue::GetArraySafe() const { + if (Type != JSON_ARRAY) + ythrow TJsonException() << "Not an array"; - return *Value.Array; - } + return *Value.Array; + } - TJsonValue::TArray& TJsonValue::GetArraySafe() { - return const_cast<TJsonValue::TArray&>(const_cast<const TJsonValue*>(this)->GetArraySafe()); - } + TJsonValue::TArray& TJsonValue::GetArraySafe() { + return const_cast<TJsonValue::TArray&>(const_cast<const TJsonValue*>(this)->GetArraySafe()); + } - bool TJsonValue::GetBooleanRobust() const noexcept { - switch (Type) { - case JSON_ARRAY: - return !Value.Array->empty(); - case JSON_MAP: - return !Value.Map->empty(); - case JSON_INTEGER: - case JSON_UINTEGER: - case JSON_DOUBLE: - return GetIntegerRobust(); - case JSON_STRING: - return GetIntegerRobust() || IsTrue(Value.String); - case JSON_NULL: - case JSON_UNDEFINED: - default: + bool TJsonValue::GetBooleanRobust() const noexcept { + switch (Type) { + case JSON_ARRAY: + return !Value.Array->empty(); + case JSON_MAP: + return !Value.Map->empty(); + case JSON_INTEGER: + case JSON_UINTEGER: + case JSON_DOUBLE: + return GetIntegerRobust(); + case JSON_STRING: + return GetIntegerRobust() || IsTrue(Value.String); + case JSON_NULL: + case JSON_UNDEFINED: + default: return false; - case JSON_BOOLEAN: - return Value.Boolean; - } - } - - long long TJsonValue::GetIntegerRobust() const noexcept { - switch (Type) { - case JSON_ARRAY: - return Value.Array->size(); - case JSON_MAP: - return Value.Map->size(); - case JSON_BOOLEAN: - return Value.Boolean; - case JSON_DOUBLE: - return GetDoubleRobust(); - case JSON_STRING: - try { - i64 res = 0; - if (Value.String && TryFromString(Value.String, res)) { - return res; - } - } catch (const yexception&) { - } - return 0; - case JSON_NULL: - case JSON_UNDEFINED: - default: - return 0; - case JSON_INTEGER: - case JSON_UINTEGER: - return Value.Integer; - } - } - - unsigned long long TJsonValue::GetUIntegerRobust() const noexcept { - switch (Type) { - case JSON_ARRAY: - return Value.Array->size(); - case JSON_MAP: - return Value.Map->size(); - case JSON_BOOLEAN: - return Value.Boolean; - case JSON_DOUBLE: - return GetDoubleRobust(); - case JSON_STRING: - try { - ui64 res = 0; - if (Value.String && TryFromString(Value.String, res)) { - return res; - } - } catch (const yexception&) { - } - return 0; - case JSON_NULL: - case JSON_UNDEFINED: - default: - return 0; - case JSON_INTEGER: - case JSON_UINTEGER: - return Value.UInteger; - } - } - - double TJsonValue::GetDoubleRobust() const noexcept { - switch (Type) { - case JSON_ARRAY: - return Value.Array->size(); - case JSON_MAP: - return Value.Map->size(); - case JSON_BOOLEAN: - return Value.Boolean; - case JSON_INTEGER: - return Value.Integer; - case JSON_UINTEGER: - return Value.UInteger; - case JSON_STRING: - try { - double res = 0; - if (Value.String && TryFromString(Value.String, res)) { - return res; - } - } catch (const yexception&) { - } - return 0; - case JSON_NULL: - case JSON_UNDEFINED: - default: - return 0; - case JSON_DOUBLE: - return Value.Double; - } - } - - TString TJsonValue::GetStringRobust() const { - switch (Type) { - case JSON_ARRAY: - case JSON_MAP: - case JSON_BOOLEAN: - case JSON_DOUBLE: - case JSON_INTEGER: - case JSON_UINTEGER: - case JSON_NULL: - case JSON_UNDEFINED: - default: { - NJsonWriter::TBuf sout; - sout.WriteJsonValue(this); - return sout.Str(); - } - case JSON_STRING: - return Value.String; - } - } - - bool TJsonValue::GetBoolean(bool* value) const noexcept { - if (Type != JSON_BOOLEAN) - return false; - - *value = Value.Boolean; - return true; - } - - bool TJsonValue::GetInteger(long long* value) const noexcept { - if (!IsInteger()) - return false; - - *value = GetInteger(); - return true; - } - - bool TJsonValue::GetUInteger(unsigned long long* value) const noexcept { - if (!IsUInteger()) - return false; - - *value = GetUInteger(); - return true; - } - - bool TJsonValue::GetDouble(double* value) const noexcept { - if (!IsDouble()) - return false; - - *value = GetDouble(); - return true; - } - - bool TJsonValue::GetString(TString* value) const { - if (Type != JSON_STRING) - return false; - - *value = Value.String; + case JSON_BOOLEAN: + return Value.Boolean; + } + } + + long long TJsonValue::GetIntegerRobust() const noexcept { + switch (Type) { + case JSON_ARRAY: + return Value.Array->size(); + case JSON_MAP: + return Value.Map->size(); + case JSON_BOOLEAN: + return Value.Boolean; + case JSON_DOUBLE: + return GetDoubleRobust(); + case JSON_STRING: + try { + i64 res = 0; + if (Value.String && TryFromString(Value.String, res)) { + return res; + } + } catch (const yexception&) { + } + return 0; + case JSON_NULL: + case JSON_UNDEFINED: + default: + return 0; + case JSON_INTEGER: + case JSON_UINTEGER: + return Value.Integer; + } + } + + unsigned long long TJsonValue::GetUIntegerRobust() const noexcept { + switch (Type) { + case JSON_ARRAY: + return Value.Array->size(); + case JSON_MAP: + return Value.Map->size(); + case JSON_BOOLEAN: + return Value.Boolean; + case JSON_DOUBLE: + return GetDoubleRobust(); + case JSON_STRING: + try { + ui64 res = 0; + if (Value.String && TryFromString(Value.String, res)) { + return res; + } + } catch (const yexception&) { + } + return 0; + case JSON_NULL: + case JSON_UNDEFINED: + default: + return 0; + case JSON_INTEGER: + case JSON_UINTEGER: + return Value.UInteger; + } + } + + double TJsonValue::GetDoubleRobust() const noexcept { + switch (Type) { + case JSON_ARRAY: + return Value.Array->size(); + case JSON_MAP: + return Value.Map->size(); + case JSON_BOOLEAN: + return Value.Boolean; + case JSON_INTEGER: + return Value.Integer; + case JSON_UINTEGER: + return Value.UInteger; + case JSON_STRING: + try { + double res = 0; + if (Value.String && TryFromString(Value.String, res)) { + return res; + } + } catch (const yexception&) { + } + return 0; + case JSON_NULL: + case JSON_UNDEFINED: + default: + return 0; + case JSON_DOUBLE: + return Value.Double; + } + } + + TString TJsonValue::GetStringRobust() const { + switch (Type) { + case JSON_ARRAY: + case JSON_MAP: + case JSON_BOOLEAN: + case JSON_DOUBLE: + case JSON_INTEGER: + case JSON_UINTEGER: + case JSON_NULL: + case JSON_UNDEFINED: + default: { + NJsonWriter::TBuf sout; + sout.WriteJsonValue(this); + return sout.Str(); + } + case JSON_STRING: + return Value.String; + } + } + + bool TJsonValue::GetBoolean(bool* value) const noexcept { + if (Type != JSON_BOOLEAN) + return false; + + *value = Value.Boolean; + return true; + } + + bool TJsonValue::GetInteger(long long* value) const noexcept { + if (!IsInteger()) + return false; + + *value = GetInteger(); + return true; + } + + bool TJsonValue::GetUInteger(unsigned long long* value) const noexcept { + if (!IsUInteger()) + return false; + + *value = GetUInteger(); + return true; + } + + bool TJsonValue::GetDouble(double* value) const noexcept { + if (!IsDouble()) + return false; + + *value = GetDouble(); + return true; + } + + bool TJsonValue::GetString(TString* value) const { + if (Type != JSON_STRING) + return false; + + *value = Value.String; + return true; + } + + bool TJsonValue::GetMap(TJsonValue::TMapType* value) const { + if (Type != JSON_MAP) + return false; + + *value = *Value.Map; + return true; + } + + bool TJsonValue::GetArray(TJsonValue::TArray* value) const { + if (Type != JSON_ARRAY) + return false; + + *value = *Value.Array; + return true; + } + + bool TJsonValue::GetMapPointer(const TJsonValue::TMapType** value) const noexcept { + if (Type != JSON_MAP) + return false; + + *value = Value.Map; + return true; + } + + bool TJsonValue::GetArrayPointer(const TJsonValue::TArray** value) const noexcept { + if (Type != JSON_ARRAY) + return false; + + *value = Value.Array; return true; } - bool TJsonValue::GetMap(TJsonValue::TMapType* value) const { - if (Type != JSON_MAP) - return false; - - *value = *Value.Map; - return true; + bool TJsonValue::GetValue(const size_t index, TJsonValue* value) const { + const TJsonValue* tmp = nullptr; + if (GetValuePointer(index, &tmp)) { + *value = *tmp; + return true; + } + return false; } - bool TJsonValue::GetArray(TJsonValue::TArray* value) const { - if (Type != JSON_ARRAY) - return false; - - *value = *Value.Array; - return true; + bool TJsonValue::GetValue(const TStringBuf key, TJsonValue* value) const { + const TJsonValue* tmp = nullptr; + if (GetValuePointer(key, &tmp)) { + *value = *tmp; + return true; + } + return false; } - bool TJsonValue::GetMapPointer(const TJsonValue::TMapType** value) const noexcept { - if (Type != JSON_MAP) - return false; - - *value = Value.Map; - return true; - } - - bool TJsonValue::GetArrayPointer(const TJsonValue::TArray** value) const noexcept { - if (Type != JSON_ARRAY) - return false; - - *value = Value.Array; - return true; - } - - bool TJsonValue::GetValue(const size_t index, TJsonValue* value) const { - const TJsonValue* tmp = nullptr; - if (GetValuePointer(index, &tmp)) { - *value = *tmp; + bool TJsonValue::GetValuePointer(const size_t index, const TJsonValue** value) const noexcept { + if (Type == JSON_ARRAY && index < Value.Array->size()) { + *value = &(*Value.Array)[index]; return true; } - return false; + return false; } - bool TJsonValue::GetValue(const TStringBuf key, TJsonValue* value) const { - const TJsonValue* tmp = nullptr; - if (GetValuePointer(key, &tmp)) { - *value = *tmp; - return true; - } - return false; - } + bool TJsonValue::GetValuePointer(const TStringBuf key, const TJsonValue** value) const noexcept { + if (Type == JSON_MAP) { + const TMapType::const_iterator it = Value.Map->find(key); + if (it != Value.Map->end()) { + *value = &(it->second); + return true; + } + } + return false; + } - bool TJsonValue::GetValuePointer(const size_t index, const TJsonValue** value) const noexcept { - if (Type == JSON_ARRAY && index < Value.Array->size()) { - *value = &(*Value.Array)[index]; - return true; - } - return false; - } + bool TJsonValue::GetValuePointer(const TStringBuf key, TJsonValue** value) noexcept { + return static_cast<const TJsonValue*>(this)->GetValuePointer(key, const_cast<const TJsonValue**>(value)); + } - bool TJsonValue::GetValuePointer(const TStringBuf key, const TJsonValue** value) const noexcept { - if (Type == JSON_MAP) { - const TMapType::const_iterator it = Value.Map->find(key); - if (it != Value.Map->end()) { - *value = &(it->second); - return true; - } - } - return false; - } + bool TJsonValue::IsNull() const noexcept { + return Type == JSON_NULL; + } - bool TJsonValue::GetValuePointer(const TStringBuf key, TJsonValue** value) noexcept { - return static_cast<const TJsonValue*>(this)->GetValuePointer(key, const_cast<const TJsonValue**>(value)); - } - - bool TJsonValue::IsNull() const noexcept { - return Type == JSON_NULL; - } + bool TJsonValue::IsBoolean() const noexcept { + return Type == JSON_BOOLEAN; + } - bool TJsonValue::IsBoolean() const noexcept { - return Type == JSON_BOOLEAN; - } + bool TJsonValue::IsInteger() const noexcept { + switch (Type) { + case JSON_INTEGER: + return true; - bool TJsonValue::IsInteger() const noexcept { - switch (Type) { - case JSON_INTEGER: - return true; + case JSON_UINTEGER: + return (Value.UInteger <= static_cast<unsigned long long>(Max<long long>())); - case JSON_UINTEGER: - return (Value.UInteger <= static_cast<unsigned long long>(Max<long long>())); - - case JSON_DOUBLE: - return ((long long)Value.Double == Value.Double); - - default: - return false; - } + case JSON_DOUBLE: + return ((long long)Value.Double == Value.Double); + + default: + return false; + } } - bool TJsonValue::IsUInteger() const noexcept { - switch (Type) { - case JSON_UINTEGER: - return true; + bool TJsonValue::IsUInteger() const noexcept { + switch (Type) { + case JSON_UINTEGER: + return true; - case JSON_INTEGER: - return (Value.Integer >= 0); + case JSON_INTEGER: + return (Value.Integer >= 0); - case JSON_DOUBLE: - return ((unsigned long long)Value.Double == Value.Double); + case JSON_DOUBLE: + return ((unsigned long long)Value.Double == Value.Double); - default: - return false; - } + default: + return false; + } } - bool TJsonValue::IsDouble() const noexcept { - // Check whether we can convert integer to floating-point - // without precision loss. - switch (Type) { - case JSON_DOUBLE: - return true; + bool TJsonValue::IsDouble() const noexcept { + // Check whether we can convert integer to floating-point + // without precision loss. + switch (Type) { + case JSON_DOUBLE: + return true; - case JSON_INTEGER: + case JSON_INTEGER: return (1ll << std::numeric_limits<double>::digits) >= Abs(Value.Integer); - case JSON_UINTEGER: + case JSON_UINTEGER: return (1ull << std::numeric_limits<double>::digits) >= Value.UInteger; - default: - return false; - } - } - - namespace { - template <class TPtr, class T> - TPtr* CreateOrNullptr(TPtr* p, T key, std::true_type /*create*/) { - return &(*p)[key]; - } - - template <class TPtr, class T> - TPtr* CreateOrNullptr(const TPtr* p, T key, std::false_type /*create*/) noexcept { - const TPtr* const next = &(*p)[key]; - return next->IsDefined() ? const_cast<TPtr*>(next) : nullptr; - } - - template <bool Create, class TJsonPtr> - TJsonPtr GetValuePtrByPath(TJsonPtr currentJson, TStringBuf path, char delimiter) noexcept(!Create) { - static_assert( - !(Create && std::is_const<std::remove_pointer_t<TJsonPtr>>::value), - "TJsonPtr must be a `TJsonValue*` if `Create` is true"); - constexpr std::integral_constant<bool, Create> create_tag{}; - - while (!path.empty()) { - size_t index = 0; - const TStringBuf step = path.NextTok(delimiter); - if (step.size() > 2 && *step.begin() == '[' && step.back() == ']' && TryFromString(step.substr(1, step.size() - 2), index)) { - currentJson = CreateOrNullptr(currentJson, index, create_tag); - } else { - currentJson = CreateOrNullptr(currentJson, step, create_tag); - } - - if (!currentJson) { - return nullptr; - } + default: + return false; + } + } + + namespace { + template <class TPtr, class T> + TPtr* CreateOrNullptr(TPtr* p, T key, std::true_type /*create*/) { + return &(*p)[key]; + } + + template <class TPtr, class T> + TPtr* CreateOrNullptr(const TPtr* p, T key, std::false_type /*create*/) noexcept { + const TPtr* const next = &(*p)[key]; + return next->IsDefined() ? const_cast<TPtr*>(next) : nullptr; + } + + template <bool Create, class TJsonPtr> + TJsonPtr GetValuePtrByPath(TJsonPtr currentJson, TStringBuf path, char delimiter) noexcept(!Create) { + static_assert( + !(Create && std::is_const<std::remove_pointer_t<TJsonPtr>>::value), + "TJsonPtr must be a `TJsonValue*` if `Create` is true"); + constexpr std::integral_constant<bool, Create> create_tag{}; + + while (!path.empty()) { + size_t index = 0; + const TStringBuf step = path.NextTok(delimiter); + if (step.size() > 2 && *step.begin() == '[' && step.back() == ']' && TryFromString(step.substr(1, step.size() - 2), index)) { + currentJson = CreateOrNullptr(currentJson, index, create_tag); + } else { + currentJson = CreateOrNullptr(currentJson, step, create_tag); + } + + if (!currentJson) { + return nullptr; + } } - return currentJson; + return currentJson; } - } // anonymous namespace + } // anonymous namespace - bool TJsonValue::GetValueByPath(const TStringBuf path, TJsonValue& result, char delimiter) const { - const TJsonValue* const ptr = GetValuePtrByPath<false>(this, path, delimiter); - if (ptr) { - result = *ptr; - return true; - } - return false; + bool TJsonValue::GetValueByPath(const TStringBuf path, TJsonValue& result, char delimiter) const { + const TJsonValue* const ptr = GetValuePtrByPath<false>(this, path, delimiter); + if (ptr) { + result = *ptr; + return true; + } + return false; } - bool TJsonValue::SetValueByPath(const TStringBuf path, const TJsonValue& value, char delimiter) { - TJsonValue* const ptr = GetValuePtrByPath<true>(this, path, delimiter); - if (ptr) { - *ptr = value; - return true; - } - return false; + bool TJsonValue::SetValueByPath(const TStringBuf path, const TJsonValue& value, char delimiter) { + TJsonValue* const ptr = GetValuePtrByPath<true>(this, path, delimiter); + if (ptr) { + *ptr = value; + return true; + } + return false; } - bool TJsonValue::SetValueByPath(const TStringBuf path, TJsonValue&& value, char delimiter) { - TJsonValue* const ptr = GetValuePtrByPath<true>(this, path, delimiter); - if (ptr) { - *ptr = std::move(value); - return true; - } - return false; + bool TJsonValue::SetValueByPath(const TStringBuf path, TJsonValue&& value, char delimiter) { + TJsonValue* const ptr = GetValuePtrByPath<true>(this, path, delimiter); + if (ptr) { + *ptr = std::move(value); + return true; + } + return false; } - const TJsonValue* TJsonValue::GetValueByPath(const TStringBuf key, char delim) const noexcept { - return GetValuePtrByPath<false>(this, key, delim); + const TJsonValue* TJsonValue::GetValueByPath(const TStringBuf key, char delim) const noexcept { + return GetValuePtrByPath<false>(this, key, delim); } - TJsonValue* TJsonValue::GetValueByPath(const TStringBuf key, char delim) noexcept { - return GetValuePtrByPath<false>(this, key, delim); + TJsonValue* TJsonValue::GetValueByPath(const TStringBuf key, char delim) noexcept { + return GetValuePtrByPath<false>(this, key, delim); } - void TJsonValue::DoScan(const TString& path, TJsonValue* parent, IScanCallback& callback) { - if (!callback.Do(path, parent, *this)) { - return; + void TJsonValue::DoScan(const TString& path, TJsonValue* parent, IScanCallback& callback) { + if (!callback.Do(path, parent, *this)) { + return; } - - if (Type == JSON_MAP) { - for (auto&& i : *Value.Map) { + + if (Type == JSON_MAP) { + for (auto&& i : *Value.Map) { i.second.DoScan(!!path ? TString::Join(path, ".", i.first) : i.first, this, callback); - } - } else if (Type == JSON_ARRAY) { - for (ui32 i = 0; i < Value.Array->size(); ++i) { + } + } else if (Type == JSON_ARRAY) { + for (ui32 i = 0; i < Value.Array->size(); ++i) { (*Value.Array)[i].DoScan(TString::Join(path, "[", ToString(i), "]"), this, callback); - } + } } } - void TJsonValue::Scan(IScanCallback& callback) { - DoScan("", nullptr, callback); + void TJsonValue::Scan(IScanCallback& callback) { + DoScan("", nullptr, callback); } - bool TJsonValue::IsString() const noexcept { - return Type == JSON_STRING; + bool TJsonValue::IsString() const noexcept { + return Type == JSON_STRING; } - bool TJsonValue::IsMap() const noexcept { - return Type == JSON_MAP; + bool TJsonValue::IsMap() const noexcept { + return Type == JSON_MAP; } - bool TJsonValue::IsArray() const noexcept { - return Type == JSON_ARRAY; + bool TJsonValue::IsArray() const noexcept { + return Type == JSON_ARRAY; } - bool TJsonValue::Has(const TStringBuf& key) const noexcept { + bool TJsonValue::Has(const TStringBuf& key) const noexcept { return Type == JSON_MAP && Value.Map->contains(key); } - bool TJsonValue::Has(size_t key) const noexcept { - return Type == JSON_ARRAY && Value.Array->size() > key; - } - - bool TJsonValue::operator==(const TJsonValue& rhs) const { - switch (Type) { - case JSON_UNDEFINED: { - return (rhs.GetType() == JSON_UNDEFINED); - } - - case JSON_NULL: { - return rhs.IsNull(); - } - - case JSON_BOOLEAN: { - return (rhs.IsBoolean() && Value.Boolean == rhs.Value.Boolean); - } - - case JSON_INTEGER: { - return (rhs.IsInteger() && GetInteger() == rhs.GetInteger()); - } - - case JSON_UINTEGER: { - return (rhs.IsUInteger() && GetUInteger() == rhs.GetUInteger()); - } - - case JSON_STRING: { - return (rhs.IsString() && Value.String == rhs.Value.String); - } - - case JSON_DOUBLE: { - return (rhs.IsDouble() && fabs(GetDouble() - rhs.GetDouble()) <= FLT_EPSILON); - } - - case JSON_MAP: - return AreJsonMapsEqual(*this, rhs); - - case JSON_ARRAY: - return AreJsonArraysEqual(*this, rhs); - - default: - Y_ASSERT(false && "Unknown type."); - return false; - } - } - - void TJsonValue::SwapWithUndefined(TJsonValue& output) noexcept { - if (Type == JSON_STRING) { - static_assert(std::is_nothrow_move_constructible<TString>::value, "noexcept violation! Add some try {} catch (...) logic"); - new (&output.Value.String) TString(std::move(Value.String)); - Value.String.~TString(); - } else { - std::memcpy(&output.Value, &Value, sizeof(Value)); - } - - output.Type = Type; - Type = JSON_UNDEFINED; - } - - void TJsonValue::Swap(TJsonValue& rhs) noexcept { - TJsonValue tmp(std::move(*this)); - rhs.SwapWithUndefined(*this); - tmp.SwapWithUndefined(rhs); - } + bool TJsonValue::Has(size_t key) const noexcept { + return Type == JSON_ARRAY && Value.Array->size() > key; + } + + bool TJsonValue::operator==(const TJsonValue& rhs) const { + switch (Type) { + case JSON_UNDEFINED: { + return (rhs.GetType() == JSON_UNDEFINED); + } + + case JSON_NULL: { + return rhs.IsNull(); + } + + case JSON_BOOLEAN: { + return (rhs.IsBoolean() && Value.Boolean == rhs.Value.Boolean); + } + + case JSON_INTEGER: { + return (rhs.IsInteger() && GetInteger() == rhs.GetInteger()); + } + + case JSON_UINTEGER: { + return (rhs.IsUInteger() && GetUInteger() == rhs.GetUInteger()); + } + + case JSON_STRING: { + return (rhs.IsString() && Value.String == rhs.Value.String); + } + + case JSON_DOUBLE: { + return (rhs.IsDouble() && fabs(GetDouble() - rhs.GetDouble()) <= FLT_EPSILON); + } + + case JSON_MAP: + return AreJsonMapsEqual(*this, rhs); + + case JSON_ARRAY: + return AreJsonArraysEqual(*this, rhs); + + default: + Y_ASSERT(false && "Unknown type."); + return false; + } + } + + void TJsonValue::SwapWithUndefined(TJsonValue& output) noexcept { + if (Type == JSON_STRING) { + static_assert(std::is_nothrow_move_constructible<TString>::value, "noexcept violation! Add some try {} catch (...) logic"); + new (&output.Value.String) TString(std::move(Value.String)); + Value.String.~TString(); + } else { + std::memcpy(&output.Value, &Value, sizeof(Value)); + } + + output.Type = Type; + Type = JSON_UNDEFINED; + } + + void TJsonValue::Swap(TJsonValue& rhs) noexcept { + TJsonValue tmp(std::move(*this)); + rhs.SwapWithUndefined(*this); + tmp.SwapWithUndefined(rhs); + } void TJsonValue::Save(IOutputStream* s) const { ::Save(s, static_cast<ui8>(Type)); @@ -1051,43 +1051,43 @@ namespace NJson { } } - //**************************************************************** + //**************************************************************** - bool GetMapPointer(const TJsonValue& jv, const size_t index, const TJsonValue::TMapType** value) { - const TJsonValue* v; - if (!jv.GetValuePointer(index, &v) || !v->IsMap()) - return false; + bool GetMapPointer(const TJsonValue& jv, const size_t index, const TJsonValue::TMapType** value) { + const TJsonValue* v; + if (!jv.GetValuePointer(index, &v) || !v->IsMap()) + return false; - *value = &v->GetMap(); - return true; - } + *value = &v->GetMap(); + return true; + } - bool GetArrayPointer(const TJsonValue& jv, const size_t index, const TJsonValue::TArray** value) { - const TJsonValue* v; - if (!jv.GetValuePointer(index, &v) || !v->IsArray()) - return false; + bool GetArrayPointer(const TJsonValue& jv, const size_t index, const TJsonValue::TArray** value) { + const TJsonValue* v; + if (!jv.GetValuePointer(index, &v) || !v->IsArray()) + return false; - *value = &v->GetArray(); - return true; - } + *value = &v->GetArray(); + return true; + } - bool GetMapPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TMapType** value) { - const TJsonValue* v; - if (!jv.GetValuePointer(key, &v) || !v->IsMap()) - return false; + bool GetMapPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TMapType** value) { + const TJsonValue* v; + if (!jv.GetValuePointer(key, &v) || !v->IsMap()) + return false; - *value = &v->GetMap(); - return true; - } + *value = &v->GetMap(); + return true; + } - bool GetArrayPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TArray** value) { - const TJsonValue* v; - if (!jv.GetValuePointer(key, &v) || !v->IsArray()) - return false; + bool GetArrayPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TArray** value) { + const TJsonValue* v; + if (!jv.GetValuePointer(key, &v) || !v->IsArray()) + return false; - *value = &v->GetArray(); - return true; - } + *value = &v->GetArray(); + return true; + } void TJsonValue::BackChecks() const { if (Type != JSON_ARRAY) @@ -1098,8 +1098,8 @@ namespace NJson { } } -template <> +template <> void Out<NJson::TJsonValue>(IOutputStream& out, const NJson::TJsonValue& v) { NJsonWriter::TBuf buf(NJsonWriter::HEM_DONT_ESCAPE_HTML, &out); buf.WriteJsonValue(&v); -} +} diff --git a/library/cpp/json/writer/json_value.h b/library/cpp/json/writer/json_value.h index 3f0f50bc4c..721453f001 100644 --- a/library/cpp/json/writer/json_value.h +++ b/library/cpp/json/writer/json_value.h @@ -1,189 +1,189 @@ #pragma once #include <library/cpp/json/common/defs.h> - + #include <util/generic/string.h> #include <util/generic/hash.h> #include <util/generic/vector.h> -#include <util/generic/deque.h> +#include <util/generic/deque.h> #include <util/generic/utility.h> #include <util/generic/yexception.h> namespace NJson { - enum EJsonValueType { - JSON_UNDEFINED /* "Undefined" */, - JSON_NULL /* "Null" */, - JSON_BOOLEAN /* "Boolean" */, - JSON_INTEGER /* "Integer" */, - JSON_DOUBLE /* "Double" */, - JSON_STRING /* "String" */, - JSON_MAP /* "Map" */, - JSON_ARRAY /* "Array" */, - JSON_UINTEGER /* "UInteger" */ - }; - - class TJsonValue; - - class IScanCallback { - public: + enum EJsonValueType { + JSON_UNDEFINED /* "Undefined" */, + JSON_NULL /* "Null" */, + JSON_BOOLEAN /* "Boolean" */, + JSON_INTEGER /* "Integer" */, + JSON_DOUBLE /* "Double" */, + JSON_STRING /* "String" */, + JSON_MAP /* "Map" */, + JSON_ARRAY /* "Array" */, + JSON_UINTEGER /* "UInteger" */ + }; + + class TJsonValue; + + class IScanCallback { + public: virtual ~IScanCallback() = default; - virtual bool Do(const TString& path, TJsonValue* parent, TJsonValue& value) = 0; - }; - - class TJsonValue { - void Clear() noexcept; - - public: - typedef THashMap<TString, TJsonValue> TMapType; - typedef TDeque<TJsonValue> TArray; - - TJsonValue() noexcept = default; - TJsonValue(EJsonValueType type); - TJsonValue(bool value) noexcept; - TJsonValue(int value) noexcept; - TJsonValue(unsigned int value) noexcept; - TJsonValue(long value) noexcept; - TJsonValue(unsigned long value) noexcept; - TJsonValue(long long value) noexcept; - TJsonValue(unsigned long long value) noexcept; - TJsonValue(double value) noexcept; + virtual bool Do(const TString& path, TJsonValue* parent, TJsonValue& value) = 0; + }; + + class TJsonValue { + void Clear() noexcept; + + public: + typedef THashMap<TString, TJsonValue> TMapType; + typedef TDeque<TJsonValue> TArray; + + TJsonValue() noexcept = default; + TJsonValue(EJsonValueType type); + TJsonValue(bool value) noexcept; + TJsonValue(int value) noexcept; + TJsonValue(unsigned int value) noexcept; + TJsonValue(long value) noexcept; + TJsonValue(unsigned long value) noexcept; + TJsonValue(long long value) noexcept; + TJsonValue(unsigned long long value) noexcept; + TJsonValue(double value) noexcept; TJsonValue(TString value); - TJsonValue(const char* value); - template <class T> - TJsonValue(const T*) = delete; - TJsonValue(TStringBuf value); - - TJsonValue(const std::string& s) - : TJsonValue(TStringBuf(s)) - { - } - - TJsonValue(const TJsonValue& vval); - TJsonValue(TJsonValue&& vval) noexcept; - - TJsonValue& operator=(const TJsonValue& val); - TJsonValue& operator=(TJsonValue&& val) noexcept; - - ~TJsonValue() { - Clear(); - } - - EJsonValueType GetType() const noexcept; - TJsonValue& SetType(EJsonValueType type); - - TJsonValue& SetValue(const TJsonValue& value); - TJsonValue& SetValue(TJsonValue&& value); - - // for Map - TJsonValue& InsertValue(const TString& key, const TJsonValue& value); - TJsonValue& InsertValue(TStringBuf key, const TJsonValue& value); - TJsonValue& InsertValue(const char* key, const TJsonValue& value); - TJsonValue& InsertValue(const TString& key, TJsonValue&& value); - TJsonValue& InsertValue(TStringBuf key, TJsonValue&& value); - TJsonValue& InsertValue(const char* key, TJsonValue&& value); - - // for Array - TJsonValue& AppendValue(const TJsonValue& value); - TJsonValue& AppendValue(TJsonValue&& value); - TJsonValue& Back(); + TJsonValue(const char* value); + template <class T> + TJsonValue(const T*) = delete; + TJsonValue(TStringBuf value); + + TJsonValue(const std::string& s) + : TJsonValue(TStringBuf(s)) + { + } + + TJsonValue(const TJsonValue& vval); + TJsonValue(TJsonValue&& vval) noexcept; + + TJsonValue& operator=(const TJsonValue& val); + TJsonValue& operator=(TJsonValue&& val) noexcept; + + ~TJsonValue() { + Clear(); + } + + EJsonValueType GetType() const noexcept; + TJsonValue& SetType(EJsonValueType type); + + TJsonValue& SetValue(const TJsonValue& value); + TJsonValue& SetValue(TJsonValue&& value); + + // for Map + TJsonValue& InsertValue(const TString& key, const TJsonValue& value); + TJsonValue& InsertValue(TStringBuf key, const TJsonValue& value); + TJsonValue& InsertValue(const char* key, const TJsonValue& value); + TJsonValue& InsertValue(const TString& key, TJsonValue&& value); + TJsonValue& InsertValue(TStringBuf key, TJsonValue&& value); + TJsonValue& InsertValue(const char* key, TJsonValue&& value); + + // for Array + TJsonValue& AppendValue(const TJsonValue& value); + TJsonValue& AppendValue(TJsonValue&& value); + TJsonValue& Back(); const TJsonValue& Back() const; - bool GetValueByPath(TStringBuf path, TJsonValue& result, char delimiter = '.') const; - bool SetValueByPath(TStringBuf path, const TJsonValue& value, char delimiter = '.'); - bool SetValueByPath(TStringBuf path, TJsonValue&& value, char delimiter = '.'); - - // returns NULL on failure - const TJsonValue* GetValueByPath(TStringBuf path, char delimiter = '.') const noexcept; - TJsonValue* GetValueByPath(TStringBuf path, char delimiter = '.') noexcept; - - void EraseValue(TStringBuf key); - void EraseValue(size_t index); - - TJsonValue& operator[](size_t idx); - TJsonValue& operator[](const TStringBuf& key); - const TJsonValue& operator[](size_t idx) const noexcept; - const TJsonValue& operator[](const TStringBuf& key) const noexcept; - - bool GetBoolean() const; - long long GetInteger() const; - unsigned long long GetUInteger() const; - double GetDouble() const; - const TString& GetString() const; - const TMapType& GetMap() const; - const TArray& GetArray() const; - - //throwing TJsonException possible - bool GetBooleanSafe() const; - long long GetIntegerSafe() const; - unsigned long long GetUIntegerSafe() const; - double GetDoubleSafe() const; - const TString& GetStringSafe() const; - const TMapType& GetMapSafe() const; - TMapType& GetMapSafe(); - const TArray& GetArraySafe() const; - TArray& GetArraySafe(); - - bool GetBooleanSafe(bool defaultValue) const; - long long GetIntegerSafe(long long defaultValue) const; - unsigned long long GetUIntegerSafe(unsigned long long defaultValue) const; - double GetDoubleSafe(double defaultValue) const; - TString GetStringSafe(const TString& defaultValue) const; - - bool GetBooleanRobust() const noexcept; - long long GetIntegerRobust() const noexcept; - unsigned long long GetUIntegerRobust() const noexcept; - double GetDoubleRobust() const noexcept; - TString GetStringRobust() const; - - // Exception-free accessors - bool GetBoolean(bool* value) const noexcept; - bool GetInteger(long long* value) const noexcept; - bool GetUInteger(unsigned long long* value) const noexcept; - bool GetDouble(double* value) const noexcept; - bool GetMapPointer(const TMapType** value) const noexcept; - bool GetArrayPointer(const TArray** value) const noexcept; - - bool GetString(TString* value) const; - bool GetMap(TMapType* value) const; - bool GetArray(TArray* value) const; - bool GetValue(size_t index, TJsonValue* value) const; - bool GetValue(TStringBuf key, TJsonValue* value) const; - bool GetValuePointer(size_t index, const TJsonValue** value) const noexcept; - bool GetValuePointer(TStringBuf key, const TJsonValue** value) const noexcept; - bool GetValuePointer(TStringBuf key, TJsonValue** value) noexcept; - - // Checking for defined non-null value - bool IsDefined() const noexcept { - return Type != JSON_UNDEFINED && Type != JSON_NULL; - } - - bool IsNull() const noexcept; - bool IsBoolean() const noexcept; - bool IsDouble() const noexcept; - bool IsString() const noexcept; - bool IsMap() const noexcept; - bool IsArray() const noexcept; - - /// @return true if JSON_INTEGER or (JSON_UINTEGER and Value <= Max<long long>) - bool IsInteger() const noexcept; - - /// @return true if JSON_UINTEGER or (JSON_INTEGER and Value >= 0) - bool IsUInteger() const noexcept; - - bool Has(const TStringBuf& key) const noexcept; - bool Has(size_t key) const noexcept; - - void Scan(IScanCallback& callback); - - /// Non-robust comparison. - bool operator==(const TJsonValue& rhs) const; - - bool operator!=(const TJsonValue& rhs) const { - return !(*this == rhs); - } - - void Swap(TJsonValue& rhs) noexcept; + bool GetValueByPath(TStringBuf path, TJsonValue& result, char delimiter = '.') const; + bool SetValueByPath(TStringBuf path, const TJsonValue& value, char delimiter = '.'); + bool SetValueByPath(TStringBuf path, TJsonValue&& value, char delimiter = '.'); + + // returns NULL on failure + const TJsonValue* GetValueByPath(TStringBuf path, char delimiter = '.') const noexcept; + TJsonValue* GetValueByPath(TStringBuf path, char delimiter = '.') noexcept; + + void EraseValue(TStringBuf key); + void EraseValue(size_t index); + + TJsonValue& operator[](size_t idx); + TJsonValue& operator[](const TStringBuf& key); + const TJsonValue& operator[](size_t idx) const noexcept; + const TJsonValue& operator[](const TStringBuf& key) const noexcept; + + bool GetBoolean() const; + long long GetInteger() const; + unsigned long long GetUInteger() const; + double GetDouble() const; + const TString& GetString() const; + const TMapType& GetMap() const; + const TArray& GetArray() const; + + //throwing TJsonException possible + bool GetBooleanSafe() const; + long long GetIntegerSafe() const; + unsigned long long GetUIntegerSafe() const; + double GetDoubleSafe() const; + const TString& GetStringSafe() const; + const TMapType& GetMapSafe() const; + TMapType& GetMapSafe(); + const TArray& GetArraySafe() const; + TArray& GetArraySafe(); + + bool GetBooleanSafe(bool defaultValue) const; + long long GetIntegerSafe(long long defaultValue) const; + unsigned long long GetUIntegerSafe(unsigned long long defaultValue) const; + double GetDoubleSafe(double defaultValue) const; + TString GetStringSafe(const TString& defaultValue) const; + + bool GetBooleanRobust() const noexcept; + long long GetIntegerRobust() const noexcept; + unsigned long long GetUIntegerRobust() const noexcept; + double GetDoubleRobust() const noexcept; + TString GetStringRobust() const; + + // Exception-free accessors + bool GetBoolean(bool* value) const noexcept; + bool GetInteger(long long* value) const noexcept; + bool GetUInteger(unsigned long long* value) const noexcept; + bool GetDouble(double* value) const noexcept; + bool GetMapPointer(const TMapType** value) const noexcept; + bool GetArrayPointer(const TArray** value) const noexcept; + + bool GetString(TString* value) const; + bool GetMap(TMapType* value) const; + bool GetArray(TArray* value) const; + bool GetValue(size_t index, TJsonValue* value) const; + bool GetValue(TStringBuf key, TJsonValue* value) const; + bool GetValuePointer(size_t index, const TJsonValue** value) const noexcept; + bool GetValuePointer(TStringBuf key, const TJsonValue** value) const noexcept; + bool GetValuePointer(TStringBuf key, TJsonValue** value) noexcept; + + // Checking for defined non-null value + bool IsDefined() const noexcept { + return Type != JSON_UNDEFINED && Type != JSON_NULL; + } + + bool IsNull() const noexcept; + bool IsBoolean() const noexcept; + bool IsDouble() const noexcept; + bool IsString() const noexcept; + bool IsMap() const noexcept; + bool IsArray() const noexcept; + + /// @return true if JSON_INTEGER or (JSON_UINTEGER and Value <= Max<long long>) + bool IsInteger() const noexcept; + + /// @return true if JSON_UINTEGER or (JSON_INTEGER and Value >= 0) + bool IsUInteger() const noexcept; + + bool Has(const TStringBuf& key) const noexcept; + bool Has(size_t key) const noexcept; + + void Scan(IScanCallback& callback); + + /// Non-robust comparison. + bool operator==(const TJsonValue& rhs) const; + + bool operator!=(const TJsonValue& rhs) const { + return !(*this == rhs); + } + + void Swap(TJsonValue& rhs) noexcept; // save using util/ysaveload.h serialization (not to JSON stream) void Save(IOutputStream* s) const; @@ -191,28 +191,28 @@ namespace NJson { // load using util/ysaveload.h serialization (not as JSON stream) void Load(IInputStream* s); - static const TJsonValue UNDEFINED; + static const TJsonValue UNDEFINED; - private: - EJsonValueType Type = JSON_UNDEFINED; - union TValueUnion { - bool Boolean; - long long Integer; - unsigned long long UInteger; - double Double; - TString String; - TMapType* Map; - TArray* Array; + private: + EJsonValueType Type = JSON_UNDEFINED; + union TValueUnion { + bool Boolean; + long long Integer; + unsigned long long UInteger; + double Double; + TString String; + TMapType* Map; + TArray* Array; - TValueUnion() noexcept { + TValueUnion() noexcept { Zero(*this); - } - ~TValueUnion() noexcept { - } - }; - TValueUnion Value; - void DoScan(const TString& path, TJsonValue* parent, IScanCallback& callback); - void SwapWithUndefined(TJsonValue& output) noexcept; + } + ~TValueUnion() noexcept { + } + }; + TValueUnion Value; + void DoScan(const TString& path, TJsonValue* parent, IScanCallback& callback); + void SwapWithUndefined(TJsonValue& output) noexcept; /** @throw yexception if Back shouldn't be called on the object. @@ -220,51 +220,51 @@ namespace NJson { void BackChecks() const; }; - inline bool GetBoolean(const TJsonValue& jv, size_t index, bool* value) noexcept { - return jv[index].GetBoolean(value); - } + inline bool GetBoolean(const TJsonValue& jv, size_t index, bool* value) noexcept { + return jv[index].GetBoolean(value); + } - inline bool GetInteger(const TJsonValue& jv, size_t index, long long* value) noexcept { - return jv[index].GetInteger(value); - } + inline bool GetInteger(const TJsonValue& jv, size_t index, long long* value) noexcept { + return jv[index].GetInteger(value); + } - inline bool GetUInteger(const TJsonValue& jv, size_t index, unsigned long long* value) noexcept { - return jv[index].GetUInteger(value); - } + inline bool GetUInteger(const TJsonValue& jv, size_t index, unsigned long long* value) noexcept { + return jv[index].GetUInteger(value); + } - inline bool GetDouble(const TJsonValue& jv, size_t index, double* value) noexcept { - return jv[index].GetDouble(value); - } + inline bool GetDouble(const TJsonValue& jv, size_t index, double* value) noexcept { + return jv[index].GetDouble(value); + } - inline bool GetString(const TJsonValue& jv, size_t index, TString* value) { - return jv[index].GetString(value); - } + inline bool GetString(const TJsonValue& jv, size_t index, TString* value) { + return jv[index].GetString(value); + } - bool GetMapPointer(const TJsonValue& jv, size_t index, const TJsonValue::TMapType** value); - bool GetArrayPointer(const TJsonValue& jv, size_t index, const TJsonValue::TArray** value); + bool GetMapPointer(const TJsonValue& jv, size_t index, const TJsonValue::TMapType** value); + bool GetArrayPointer(const TJsonValue& jv, size_t index, const TJsonValue::TArray** value); - inline bool GetBoolean(const TJsonValue& jv, TStringBuf key, bool* value) noexcept { - return jv[key].GetBoolean(value); - } + inline bool GetBoolean(const TJsonValue& jv, TStringBuf key, bool* value) noexcept { + return jv[key].GetBoolean(value); + } - inline bool GetInteger(const TJsonValue& jv, TStringBuf key, long long* value) noexcept { - return jv[key].GetInteger(value); - } + inline bool GetInteger(const TJsonValue& jv, TStringBuf key, long long* value) noexcept { + return jv[key].GetInteger(value); + } - inline bool GetUInteger(const TJsonValue& jv, TStringBuf key, unsigned long long* value) noexcept { - return jv[key].GetUInteger(value); - } + inline bool GetUInteger(const TJsonValue& jv, TStringBuf key, unsigned long long* value) noexcept { + return jv[key].GetUInteger(value); + } - inline bool GetDouble(const TJsonValue& jv, TStringBuf key, double* value) noexcept { - return jv[key].GetDouble(value); - } + inline bool GetDouble(const TJsonValue& jv, TStringBuf key, double* value) noexcept { + return jv[key].GetDouble(value); + } - inline bool GetString(const TJsonValue& jv, TStringBuf key, TString* value) { - return jv[key].GetString(value); - } + inline bool GetString(const TJsonValue& jv, TStringBuf key, TString* value) { + return jv[key].GetString(value); + } - bool GetMapPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TMapType** value); - bool GetArrayPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TArray** value); + bool GetMapPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TMapType** value); + bool GetArrayPointer(const TJsonValue& jv, const TStringBuf key, const TJsonValue::TArray** value); class TJsonMap: public TJsonValue { public: diff --git a/library/cpp/json/writer/json_value_ut.cpp b/library/cpp/json/writer/json_value_ut.cpp index dc7f6affdf..d3f3c42bea 100644 --- a/library/cpp/json/writer/json_value_ut.cpp +++ b/library/cpp/json/writer/json_value_ut.cpp @@ -18,7 +18,7 @@ Y_UNIT_TEST_SUITE(TJsonValueTest) { TJsonValue emptyMap(JSON_MAP); UNIT_ASSERT(!undef.IsDefined()); - UNIT_ASSERT(!null.IsDefined()); // json NULL is undefined too! + UNIT_ASSERT(!null.IsDefined()); // json NULL is undefined too! UNIT_ASSERT(_false.IsDefined()); UNIT_ASSERT(zeroInt.IsDefined()); UNIT_ASSERT(zeroDouble.IsDefined()); @@ -259,7 +259,7 @@ Y_UNIT_TEST_SUITE(TJsonValueTest) { UNIT_ASSERT_EQUAL(lhs.GetValueByPath("l/a/c/e/x", '/'), NULL); UNIT_ASSERT_EQUAL(lhs.GetValueByPath("a/c/e/x", '/'), NULL); UNIT_ASSERT_EQUAL(lhs.GetValueByPath("nokey", '/'), NULL); - UNIT_ASSERT_EQUAL(*lhs.GetValueByPath("", '/'), lhs); // itself + UNIT_ASSERT_EQUAL(*lhs.GetValueByPath("", '/'), lhs); // itself TJsonValue array; TJsonValue third; @@ -325,7 +325,7 @@ Y_UNIT_TEST_SUITE(TJsonValueTest) { } { const TJsonValue* result = lhs.GetValueByPath("", '/'); - UNIT_ASSERT_EQUAL(*result, lhs); // itself + UNIT_ASSERT_EQUAL(*result, lhs); // itself } TJsonValue array; @@ -335,8 +335,8 @@ Y_UNIT_TEST_SUITE(TJsonValueTest) { third["t"] = array; UNIT_ASSERT(array.GetValueByPath("[0].e", '.')->GetStringRobust() == "f"); - UNIT_ASSERT(third.GetValueByPath("t.[0].e", '.')->GetStringRobust() == "f"); - UNIT_ASSERT(third.GetValueByPath("t.[1].c.e", '.')->GetStringRobust() == "f"); + UNIT_ASSERT(third.GetValueByPath("t.[0].e", '.')->GetStringRobust() == "f"); + UNIT_ASSERT(third.GetValueByPath("t.[1].c.e", '.')->GetStringRobust() == "f"); } Y_UNIT_TEST(EraseValueFromArray) { @@ -385,7 +385,7 @@ Y_UNIT_TEST_SUITE(TJsonValueTest) { value1.AppendValue(1); value1.AppendValue(2); src.InsertValue("key", value1); - src.InsertValue("key1", "HI!"); + src.InsertValue("key1", "HI!"); TJsonValue dst; TJsonValue value2; @@ -426,7 +426,7 @@ Y_UNIT_TEST_SUITE(TJsonValueTest) { dst.InsertValue("arr", arr2); src["arr"].AppendValue(value1); - for (auto& node : src["arr"].GetArraySafe()) { + for (auto& node : src["arr"].GetArraySafe()) { node.InsertValue("yek", "eulav"); } UNIT_ASSERT(src == dst); @@ -550,7 +550,7 @@ Y_UNIT_TEST_SUITE(TJsonValueTest) { outer.AppendValue(json); } - const TJsonValue::TArray* array = nullptr; + const TJsonValue::TArray* array = nullptr; GetArrayPointer(outer, 0, &array); UNIT_ASSERT_VALUES_EQUAL((*array)[1], 2); } @@ -565,7 +565,7 @@ Y_UNIT_TEST_SUITE(TJsonValueTest) { outer.InsertValue("x", json); } - const TJsonValue::TArray* array = nullptr; + const TJsonValue::TArray* array = nullptr; GetArrayPointer(outer, "x", &array); UNIT_ASSERT_VALUES_EQUAL((*array)[1], 2); } @@ -580,7 +580,7 @@ Y_UNIT_TEST_SUITE(TJsonValueTest) { outer.AppendValue(json); } - const TJsonValue::TMapType* map = nullptr; + const TJsonValue::TMapType* map = nullptr; GetMapPointer(outer, 0, &map); UNIT_ASSERT_VALUES_EQUAL((*map).at("b"), 2); } @@ -595,7 +595,7 @@ Y_UNIT_TEST_SUITE(TJsonValueTest) { outer.InsertValue("x", json); } - const TJsonValue::TMapType* map = nullptr; + const TJsonValue::TMapType* map = nullptr; GetMapPointer(outer, "x", &map); UNIT_ASSERT_VALUES_EQUAL((*map).at("b"), 2); } @@ -617,7 +617,7 @@ Y_UNIT_TEST_SUITE(TJsonValueTest) { const char* longTestString = "Testing TJsonValue& operator=(TJsonValue&&) subpart self moving " - "after TJsonValue was constrcuted from TString&&."; + "after TJsonValue was constrcuted from TString&&."; json["hello"] = TString{longTestString}; json = std::move(json["hello"]); diff --git a/library/cpp/json/writer/ya.make b/library/cpp/json/writer/ya.make index 3989ff3504..1f1e2df08e 100644 --- a/library/cpp/json/writer/ya.make +++ b/library/cpp/json/writer/ya.make @@ -5,8 +5,8 @@ OWNER( myltsev pg ) - -PEERDIR( + +PEERDIR( library/cpp/json/common ) diff --git a/library/cpp/json/ya.make b/library/cpp/json/ya.make index d58eead8ec..d46147f269 100644 --- a/library/cpp/json/ya.make +++ b/library/cpp/json/ya.make @@ -8,7 +8,7 @@ OWNER( SRCS( json_writer.cpp json_reader.cpp - json_prettifier.cpp + json_prettifier.cpp rapidjson_helpers.cpp ) diff --git a/library/cpp/lcs/lcs_via_lis.h b/library/cpp/lcs/lcs_via_lis.h index d26733d94e..a227b01268 100644 --- a/library/cpp/lcs/lcs_via_lis.h +++ b/library/cpp/lcs/lcs_via_lis.h @@ -1,193 +1,193 @@ #pragma once #include <library/cpp/containers/paged_vector/paged_vector.h> - + #include <util/generic/ptr.h> #include <util/generic/hash.h> #include <util/generic/vector.h> -#include <util/generic/algorithm.h> +#include <util/generic/algorithm.h> #include <util/memory/pool.h> namespace NLCS { - template <typename TVal> - struct TLCSCtx { + template <typename TVal> + struct TLCSCtx { typedef TVector<ui32> TSubsequence; typedef THashMap<TVal, TSubsequence, THash<TVal>, TEqualTo<TVal>, ::TPoolAllocator> TEncounterIndex; typedef TVector<std::pair<ui32, ui32>> TLastIndex; - typedef NPagedVector::TPagedVector<TSubsequence, 4096> TCover; - - TMemoryPool Pool; - THolder<TEncounterIndex> Encounters; - TLastIndex LastIndex; - TCover Cover; - - TSubsequence ResultBuffer; - - TLCSCtx() - : Pool(16 * 1024 - 64, TMemoryPool::TExpGrow::Instance()) - { - Reset(); - } - - void Reset() { - Encounters.Reset(nullptr); - Pool.Clear(); - Encounters.Reset(new TEncounterIndex(&Pool)); - LastIndex.clear(); - Cover.clear(); - ResultBuffer.clear(); - } - }; - - namespace NPrivate { - template <typename TIt, typename TVl> - struct TSequence { - typedef TIt TIter; - typedef TVl TVal; - - const TIter Begin; - const TIter End; - const size_t Size; - - TSequence(TIter beg, TIter end) - : Begin(beg) - , End(end) - , Size(end - beg) - { - } - }; - - template <typename TVal, typename TSequence, typename TResult> - size_t MakeLCS(TSequence s1, TSequence s2, TResult* res = nullptr, TLCSCtx<TVal>* ctx = nullptr) { - typedef TLCSCtx<TVal> TCtx; - - THolder<TCtx> ctxhld; - - if (!ctx) { - ctxhld.Reset(new TCtx()); - ctx = ctxhld.Get(); - } else { - ctx->Reset(); - } - - size_t maxsize = Max(s1.Size, s2.Size); - auto& index = *(ctx->Encounters); - - for (auto it = s1.Begin; it != s1.End; ++it) { - index[*it]; - } - - for (auto it = s2.Begin; it != s2.End; ++it) { - auto hit = index.find(*it); - - if (hit != index.end()) { - hit->second.push_back(it - s2.Begin); - } - } - - if (!res) { - auto& lastindex = ctx->ResultBuffer; - lastindex.reserve(maxsize); - - for (auto it1 = s1.Begin; it1 != s1.End; ++it1) { - const auto& sub2 = index[*it1]; - - for (auto it2 = sub2.rbegin(); it2 != sub2.rend(); ++it2) { - ui32 x = *it2; - - auto lit = LowerBound(lastindex.begin(), lastindex.end(), x); - - if (lit == lastindex.end()) { - lastindex.push_back(x); - } else { - *lit = x; - } - } + typedef NPagedVector::TPagedVector<TSubsequence, 4096> TCover; + + TMemoryPool Pool; + THolder<TEncounterIndex> Encounters; + TLastIndex LastIndex; + TCover Cover; + + TSubsequence ResultBuffer; + + TLCSCtx() + : Pool(16 * 1024 - 64, TMemoryPool::TExpGrow::Instance()) + { + Reset(); + } + + void Reset() { + Encounters.Reset(nullptr); + Pool.Clear(); + Encounters.Reset(new TEncounterIndex(&Pool)); + LastIndex.clear(); + Cover.clear(); + ResultBuffer.clear(); + } + }; + + namespace NPrivate { + template <typename TIt, typename TVl> + struct TSequence { + typedef TIt TIter; + typedef TVl TVal; + + const TIter Begin; + const TIter End; + const size_t Size; + + TSequence(TIter beg, TIter end) + : Begin(beg) + , End(end) + , Size(end - beg) + { + } + }; + + template <typename TVal, typename TSequence, typename TResult> + size_t MakeLCS(TSequence s1, TSequence s2, TResult* res = nullptr, TLCSCtx<TVal>* ctx = nullptr) { + typedef TLCSCtx<TVal> TCtx; + + THolder<TCtx> ctxhld; + + if (!ctx) { + ctxhld.Reset(new TCtx()); + ctx = ctxhld.Get(); + } else { + ctx->Reset(); + } + + size_t maxsize = Max(s1.Size, s2.Size); + auto& index = *(ctx->Encounters); + + for (auto it = s1.Begin; it != s1.End; ++it) { + index[*it]; + } + + for (auto it = s2.Begin; it != s2.End; ++it) { + auto hit = index.find(*it); + + if (hit != index.end()) { + hit->second.push_back(it - s2.Begin); + } + } + + if (!res) { + auto& lastindex = ctx->ResultBuffer; + lastindex.reserve(maxsize); + + for (auto it1 = s1.Begin; it1 != s1.End; ++it1) { + const auto& sub2 = index[*it1]; + + for (auto it2 = sub2.rbegin(); it2 != sub2.rend(); ++it2) { + ui32 x = *it2; + + auto lit = LowerBound(lastindex.begin(), lastindex.end(), x); + + if (lit == lastindex.end()) { + lastindex.push_back(x); + } else { + *lit = x; + } + } + } + + return lastindex.size(); + } else { + auto& lastindex = ctx->LastIndex; + auto& cover = ctx->Cover; + + lastindex.reserve(maxsize); + + for (auto it1 = s1.Begin; it1 != s1.End; ++it1) { + const auto& sub2 = index[*it1]; + + for (auto it2 = sub2.rbegin(); it2 != sub2.rend(); ++it2) { + ui32 x = *it2; + + auto lit = LowerBound(lastindex.begin(), lastindex.end(), std::make_pair(x, (ui32)0u)); + + if (lit == lastindex.end()) { + lastindex.push_back(std::make_pair(x, cover.size())); + cover.emplace_back(); + cover.back().push_back(x); + } else { + *lit = std::make_pair(x, lit->second); + cover[lit->second].push_back(x); + } + } } - return lastindex.size(); - } else { - auto& lastindex = ctx->LastIndex; - auto& cover = ctx->Cover; - - lastindex.reserve(maxsize); - - for (auto it1 = s1.Begin; it1 != s1.End; ++it1) { - const auto& sub2 = index[*it1]; - - for (auto it2 = sub2.rbegin(); it2 != sub2.rend(); ++it2) { - ui32 x = *it2; - - auto lit = LowerBound(lastindex.begin(), lastindex.end(), std::make_pair(x, (ui32)0u)); - - if (lit == lastindex.end()) { - lastindex.push_back(std::make_pair(x, cover.size())); - cover.emplace_back(); - cover.back().push_back(x); - } else { - *lit = std::make_pair(x, lit->second); - cover[lit->second].push_back(x); - } - } - } - - if (cover.empty()) { - return 0; - } - - std::reverse(cover.begin(), cover.end()); - - auto& resbuf = ctx->ResultBuffer; - - resbuf.push_back(cover.front().front()); - - for (auto it = cover.begin() + 1; it != cover.end(); ++it) { - auto pit = UpperBound(it->begin(), it->end(), resbuf.back(), std::greater<ui32>()); - - Y_VERIFY(pit != it->end(), " "); - - resbuf.push_back(*pit); - } - - std::reverse(resbuf.begin(), resbuf.end()); - - for (auto it = resbuf.begin(); it != resbuf.end(); ++it) { - res->push_back(*(s2.Begin + *it)); - } - - return lastindex.size(); - } - } + if (cover.empty()) { + return 0; + } + + std::reverse(cover.begin(), cover.end()); + + auto& resbuf = ctx->ResultBuffer; + + resbuf.push_back(cover.front().front()); + + for (auto it = cover.begin() + 1; it != cover.end(); ++it) { + auto pit = UpperBound(it->begin(), it->end(), resbuf.back(), std::greater<ui32>()); + + Y_VERIFY(pit != it->end(), " "); + + resbuf.push_back(*pit); + } + + std::reverse(resbuf.begin(), resbuf.end()); + + for (auto it = resbuf.begin(); it != resbuf.end(); ++it) { + res->push_back(*(s2.Begin + *it)); + } + + return lastindex.size(); + } + } } - template <typename TVal, typename TIter, typename TResult> - size_t MakeLCS(TIter beg1, TIter end1, TIter beg2, TIter end2, TResult* res = nullptr, TLCSCtx<TVal>* ctx = nullptr) { - typedef NPrivate::TSequence<TIter, TVal> TSeq; + template <typename TVal, typename TIter, typename TResult> + size_t MakeLCS(TIter beg1, TIter end1, TIter beg2, TIter end2, TResult* res = nullptr, TLCSCtx<TVal>* ctx = nullptr) { + typedef NPrivate::TSequence<TIter, TVal> TSeq; - size_t sz1 = end1 - beg1; - size_t sz2 = end2 - beg2; + size_t sz1 = end1 - beg1; + size_t sz2 = end2 - beg2; - if (sz2 > sz1) { - DoSwap(beg1, beg2); - DoSwap(end1, end2); - DoSwap(sz1, sz2); - } + if (sz2 > sz1) { + DoSwap(beg1, beg2); + DoSwap(end1, end2); + DoSwap(sz1, sz2); + } - return NPrivate::MakeLCS<TVal>(TSeq(beg1, end1), TSeq(beg2, end2), res, ctx); + return NPrivate::MakeLCS<TVal>(TSeq(beg1, end1), TSeq(beg2, end2), res, ctx); } - template <typename TVal, typename TColl, typename TRes> - size_t MakeLCS(const TColl& coll1, const TColl& coll2, TRes* res = nullptr, TLCSCtx<TVal>* ctx = nullptr) { - return MakeLCS<TVal>(coll1.begin(), coll1.end(), coll2.begin(), coll2.end(), res, ctx); - } + template <typename TVal, typename TColl, typename TRes> + size_t MakeLCS(const TColl& coll1, const TColl& coll2, TRes* res = nullptr, TLCSCtx<TVal>* ctx = nullptr) { + return MakeLCS<TVal>(coll1.begin(), coll1.end(), coll2.begin(), coll2.end(), res, ctx); + } - template <typename TVal, typename TIter> - size_t MeasureLCS(TIter beg1, TIter end1, TIter beg2, TIter end2, TLCSCtx<TVal>* ctx = nullptr) { + template <typename TVal, typename TIter> + size_t MeasureLCS(TIter beg1, TIter end1, TIter beg2, TIter end2, TLCSCtx<TVal>* ctx = nullptr) { return MakeLCS<TVal>(beg1, end1, beg2, end2, (TVector<TVal>*)nullptr, ctx); - } + } - template <typename TVal, typename TColl> - size_t MeasureLCS(const TColl& coll1, const TColl& coll2, TLCSCtx<TVal>* ctx = nullptr) { - return MeasureLCS<TVal>(coll1.begin(), coll1.end(), coll2.begin(), coll2.end(), ctx); - } + template <typename TVal, typename TColl> + size_t MeasureLCS(const TColl& coll1, const TColl& coll2, TLCSCtx<TVal>* ctx = nullptr) { + return MeasureLCS<TVal>(coll1.begin(), coll1.end(), coll2.begin(), coll2.end(), ctx); + } } diff --git a/library/cpp/lcs/lcs_via_lis_ut.cpp b/library/cpp/lcs/lcs_via_lis_ut.cpp index f6ad5152b6..a1b60a16b2 100644 --- a/library/cpp/lcs/lcs_via_lis_ut.cpp +++ b/library/cpp/lcs/lcs_via_lis_ut.cpp @@ -4,10 +4,10 @@ #include "lcs_via_lis.h" class TLCSTest: public TTestBase { - UNIT_TEST_SUITE(TLCSTest); - UNIT_TEST(LCSTest); - UNIT_TEST_SUITE_END(); - + UNIT_TEST_SUITE(TLCSTest); + UNIT_TEST(LCSTest); + UNIT_TEST_SUITE_END(); + private: size_t Length(TStringBuf s1, TStringBuf s2) { TVector<TVector<size_t>> c; @@ -49,9 +49,9 @@ private: void LCSTest() { CheckLCSString("abacx", "baabca", "bac"); - const char* m = "mama_myla_ramu"; - const char* n = "papa_lubil_mamu"; - const char* s = "aa_l_amu"; + const char* m = "mama_myla_ramu"; + const char* n = "papa_lubil_mamu"; + const char* s = "aa_l_amu"; CheckLCSString(m, n, s); CheckLCSString(n, m, s); CheckLCSString(m, m, m); @@ -59,60 +59,60 @@ private: CheckLCSString("", m, ""); CheckLCSString("", "", ""); { - const char* s1 = - "atmwuaoccmgirveexxtbkkmioclarskvicyesddirlrgflnietcagkswbvsdnxksipfndmusnuee" - "tojygyjyobdfiutsbruuspvibmywhokxsarwbyirsqqnxxnbtkmucmdafaogwmobuuhejspgurpe" - "ceokhgdubqewnyqtwwqfibydbcbxhofvcjsftamhvnbdwxdqnphcdiowplcmguxmnpcufrahjagv" - "cpjqsyqurrhyghkasnodqqktldyfomgabrxdxpubenfscamoodgpocilewqtbsncggwvkkpuasdl" - "cltxywovqjligwhjxnmmtgeuujphgtdjrnxcnmwwbgxnpiotgnhyrxuvxkxdlgmpfeyocaviuhec" - "ydecqmttjictnpptoblxqgcrsojrymakgdjvcnppinymdrlgdfpyuykwpmdeifqwupojsgmruyxs" - "qijwnxngaxnppxgusyfnaelccytxwrkhxxirhnsastdlyejslrivkrpovnhbwxcdmpqbnjthjtlj" - "wnoayakfnpcwdnlgnrghjhiianhsncbjehlsoldjykymduytyiygrdreicjvghdibyjsmxnwvrqb" - "jjkjfrtlonfarbwhovladadlbyeygvuxcutxjqhosuxbemtwsqjlvvyegsfgeladiovecfxfymct" - "ulofkcogguantmicfrhpnauetejktkhamfuwirjvlyfgjrobywbbitbnckiegbiwbtmapqrbbqws" - "wviuplyprwwqoekiuxsmwgwyuwgeurvxempguwmgtadvrkxykffjxfdyxmsibmdlqhldlfbiaegt" - "kswcmfidnrhaibuscoyukwhdtoqwlpwnfgamvfqjpfklgurcwvgsluyoyiuumrwwsqgxatxnxhil" - "ywpkeaugfaahmchjruavepvnygcmcjdnvulwcuhlolsfxcsrjciwajbhdahpldcfggubcxalqxrl" - "coaiyeawxyxujtynodhnxbhs"; - const char* s2 = - "yjufxfeiifhrmydpmsqqgjwtpcxbhqmfpnvsvsapqvsmqmugpqehbdsiiqhcrasxuhcvswcwanwb" - "knesbuhtbaitcwebsdbbxwyubjoroekjxweeqnqmydbdbgbnfymcermhpbikocpsfccwjemxjpmc" - "hkhtfaoqgvvtpipujsxesiglgnpsdwfjhcawkfpffuyltqqhdkeqwkfpqjhnjdsnxlevyvydikbr" - "hnicihnevsofgouxjcnjjknxwwgaaaikdcxmhxfowadqudrapvwlcuwatrmiweijljdehxiwqrnq" - "tnhgukbwmadcjpfnxtswhmwnvvnwsllkyshfobrdmukswpgwunysxpnnlmccolvqyjsvagulpcda" - "ctsnyjleqgttcgpnhlnagxenuknpxiramgeshhjyoesupkcfcvvpwyweuvcwrawsgvfshppijuug" - "hdnujdqjtcdissmlnjgibdljjxntxrgytxlbgvsrrusatqelspeoyvndjifjqxqrpduwbyojjbhi" - "tmondbbnuuhpkglmfykeheddwkxjyapfniqoic"; + const char* s1 = + "atmwuaoccmgirveexxtbkkmioclarskvicyesddirlrgflnietcagkswbvsdnxksipfndmusnuee" + "tojygyjyobdfiutsbruuspvibmywhokxsarwbyirsqqnxxnbtkmucmdafaogwmobuuhejspgurpe" + "ceokhgdubqewnyqtwwqfibydbcbxhofvcjsftamhvnbdwxdqnphcdiowplcmguxmnpcufrahjagv" + "cpjqsyqurrhyghkasnodqqktldyfomgabrxdxpubenfscamoodgpocilewqtbsncggwvkkpuasdl" + "cltxywovqjligwhjxnmmtgeuujphgtdjrnxcnmwwbgxnpiotgnhyrxuvxkxdlgmpfeyocaviuhec" + "ydecqmttjictnpptoblxqgcrsojrymakgdjvcnppinymdrlgdfpyuykwpmdeifqwupojsgmruyxs" + "qijwnxngaxnppxgusyfnaelccytxwrkhxxirhnsastdlyejslrivkrpovnhbwxcdmpqbnjthjtlj" + "wnoayakfnpcwdnlgnrghjhiianhsncbjehlsoldjykymduytyiygrdreicjvghdibyjsmxnwvrqb" + "jjkjfrtlonfarbwhovladadlbyeygvuxcutxjqhosuxbemtwsqjlvvyegsfgeladiovecfxfymct" + "ulofkcogguantmicfrhpnauetejktkhamfuwirjvlyfgjrobywbbitbnckiegbiwbtmapqrbbqws" + "wviuplyprwwqoekiuxsmwgwyuwgeurvxempguwmgtadvrkxykffjxfdyxmsibmdlqhldlfbiaegt" + "kswcmfidnrhaibuscoyukwhdtoqwlpwnfgamvfqjpfklgurcwvgsluyoyiuumrwwsqgxatxnxhil" + "ywpkeaugfaahmchjruavepvnygcmcjdnvulwcuhlolsfxcsrjciwajbhdahpldcfggubcxalqxrl" + "coaiyeawxyxujtynodhnxbhs"; + const char* s2 = + "yjufxfeiifhrmydpmsqqgjwtpcxbhqmfpnvsvsapqvsmqmugpqehbdsiiqhcrasxuhcvswcwanwb" + "knesbuhtbaitcwebsdbbxwyubjoroekjxweeqnqmydbdbgbnfymcermhpbikocpsfccwjemxjpmc" + "hkhtfaoqgvvtpipujsxesiglgnpsdwfjhcawkfpffuyltqqhdkeqwkfpqjhnjdsnxlevyvydikbr" + "hnicihnevsofgouxjcnjjknxwwgaaaikdcxmhxfowadqudrapvwlcuwatrmiweijljdehxiwqrnq" + "tnhgukbwmadcjpfnxtswhmwnvvnwsllkyshfobrdmukswpgwunysxpnnlmccolvqyjsvagulpcda" + "ctsnyjleqgttcgpnhlnagxenuknpxiramgeshhjyoesupkcfcvvpwyweuvcwrawsgvfshppijuug" + "hdnujdqjtcdissmlnjgibdljjxntxrgytxlbgvsrrusatqelspeoyvndjifjqxqrpduwbyojjbhi" + "tmondbbnuuhpkglmfykeheddwkxjyapfniqoic"; CheckLCSLength(s1, s2, 247); } { - const char* s1 = - "ssyrtvllktbhijkyckviokukaodexiykxcvlvninoasblpuujnqnilnhmacsulaulskphkccnfop" - "jlhejqhemdpnihasouinuceugkfkaifdepvffntbkfivsddtxyslnlwiyfbbpnrwlkryetncahih" - "vqcrdvvehvrxgnitghbomkprdngrqdswypwfvpdrvqnavxnprakkgoibsxybngvenlbfcghcyldn" - "kssfuxvpvfhaawqiandxpsrkyqtiltmojmmwygevhodvsuvdojvwpvvbwpbbnerriufrwgwcjlgx" - "jcjchsfiomtkowihdtcyewknlvdeankladmdhwqxokmunttgaqdsbuyhekkxatpydfgquyxuucda" - "dllepofxoirmaablfyyibcnqkdbnsaygkqkbvupdhajfshouofnokwlbdtglrbklpgknyuiedppl" - "chxbnnmbumdtrsgwitjlmkkdxysvmsvcdulmanmsdeqkmwgfthmntdbthdthdodnubqajpfyssea" - "hwxymiyubkhhxlbmjptujiemrdljqkskdkuokvimencavihwqdaqtcljrgwvxpuegnoecobfllwu" - "upsfhjrrpiqtjlwigjkpltwfpoqxsdrojtawpaximslojqtadsactemuhpnshkgyoyouldanktcg" - "dhxdpwawabxwjhnjdmewrwtciquuiqnwdsbdvnuvjyewmjppkwvcotptmyrsqaovmaysjuvtenuy" - "orvdsssgjgcgksdwaaladocotgveuscwauawdhqlkqsjtmltvkkcfkgwpdtormkefkigkppwpvsy" - "fpblccsbyboouahotiifixsjuxlvqpqmpmkcgbvsefkqasearilhlxuqdfqqxxohhbdyiudqsree" - "btwfxdtxlaynrusgbffhltthkejitseuqyeqvhqgyobijwmspwbsisghsarji"; - const char* s2 = - "lcnygawhsrprxafwnwxnrxurpnqwwtwuciuexxvswckwopnmhhmhvdwmxtgceitqofivvavnqlmo" - "hbieyaxcysagyqcvijxyowiognsntxcdlmueoafqvqwafyhgyoewhwxvqswvwfkqohtutawwnmhr" - "pjmhyiygdekonlxhkbeaheqvnqbwhambhrrhkyfcehjfblgriumapavbcxxdqytiuylxmhtjjloa" - "fvnyhqsgalacknksxxxilgfcankxreaburvmxukmfprfpmfqgqhmirlnltkjxslumhtamtndaffh" - "ybfywiwjlafnsgsvywflsfkeeidwptigidvhnqdjiwvgkfynncyujodsjiglubptsdofftfxayno" - "txoykiivbvvipdpcgjadvbanaeljdbxxynlqqpdphirrbjqfqtnxabitncafatbosjnlimxfxlrh" - "thlfsdyfbhtwywewqubjcvskmwxwjyhesqbviflihnjfejcgjtkgicgmmcurynmdaoifojmedcqb" - "aeqalxnljvglnvyverrtosfeeuajyxkmmpjgcioqxnnqpjokxaenfoudondtahjduymwsyxpbvrf" - "kpgiavuihdliphwojgjobafhjshjnmhufciepexevtfwcybtripqmnekkirxmljumyxjpqbvmftt" - "xmfwokjskummmkaksbeoowfgjwiumpbkdujexectnghqjjuotofwuwvcgtluephymdkrscbfiaeg" - "aaypnclkstfqimisanikjxocmhcrotkntprwjbuudswuyuujfawjucwgifhqgepxeidmvcwqsqrv" - "karuvpnxhmrvdcocidgtuxasdqkwsdvijmnpmayhfiva"; + const char* s1 = + "ssyrtvllktbhijkyckviokukaodexiykxcvlvninoasblpuujnqnilnhmacsulaulskphkccnfop" + "jlhejqhemdpnihasouinuceugkfkaifdepvffntbkfivsddtxyslnlwiyfbbpnrwlkryetncahih" + "vqcrdvvehvrxgnitghbomkprdngrqdswypwfvpdrvqnavxnprakkgoibsxybngvenlbfcghcyldn" + "kssfuxvpvfhaawqiandxpsrkyqtiltmojmmwygevhodvsuvdojvwpvvbwpbbnerriufrwgwcjlgx" + "jcjchsfiomtkowihdtcyewknlvdeankladmdhwqxokmunttgaqdsbuyhekkxatpydfgquyxuucda" + "dllepofxoirmaablfyyibcnqkdbnsaygkqkbvupdhajfshouofnokwlbdtglrbklpgknyuiedppl" + "chxbnnmbumdtrsgwitjlmkkdxysvmsvcdulmanmsdeqkmwgfthmntdbthdthdodnubqajpfyssea" + "hwxymiyubkhhxlbmjptujiemrdljqkskdkuokvimencavihwqdaqtcljrgwvxpuegnoecobfllwu" + "upsfhjrrpiqtjlwigjkpltwfpoqxsdrojtawpaximslojqtadsactemuhpnshkgyoyouldanktcg" + "dhxdpwawabxwjhnjdmewrwtciquuiqnwdsbdvnuvjyewmjppkwvcotptmyrsqaovmaysjuvtenuy" + "orvdsssgjgcgksdwaaladocotgveuscwauawdhqlkqsjtmltvkkcfkgwpdtormkefkigkppwpvsy" + "fpblccsbyboouahotiifixsjuxlvqpqmpmkcgbvsefkqasearilhlxuqdfqqxxohhbdyiudqsree" + "btwfxdtxlaynrusgbffhltthkejitseuqyeqvhqgyobijwmspwbsisghsarji"; + const char* s2 = + "lcnygawhsrprxafwnwxnrxurpnqwwtwuciuexxvswckwopnmhhmhvdwmxtgceitqofivvavnqlmo" + "hbieyaxcysagyqcvijxyowiognsntxcdlmueoafqvqwafyhgyoewhwxvqswvwfkqohtutawwnmhr" + "pjmhyiygdekonlxhkbeaheqvnqbwhambhrrhkyfcehjfblgriumapavbcxxdqytiuylxmhtjjloa" + "fvnyhqsgalacknksxxxilgfcankxreaburvmxukmfprfpmfqgqhmirlnltkjxslumhtamtndaffh" + "ybfywiwjlafnsgsvywflsfkeeidwptigidvhnqdjiwvgkfynncyujodsjiglubptsdofftfxayno" + "txoykiivbvvipdpcgjadvbanaeljdbxxynlqqpdphirrbjqfqtnxabitncafatbosjnlimxfxlrh" + "thlfsdyfbhtwywewqubjcvskmwxwjyhesqbviflihnjfejcgjtkgicgmmcurynmdaoifojmedcqb" + "aeqalxnljvglnvyverrtosfeeuajyxkmmpjgcioqxnnqpjokxaenfoudondtahjduymwsyxpbvrf" + "kpgiavuihdliphwojgjobafhjshjnmhufciepexevtfwcybtripqmnekkirxmljumyxjpqbvmftt" + "xmfwokjskummmkaksbeoowfgjwiumpbkdujexectnghqjjuotofwuwvcgtluephymdkrscbfiaeg" + "aaypnclkstfqimisanikjxocmhcrotkntprwjbuudswuyuujfawjucwgifhqgepxeidmvcwqsqrv" + "karuvpnxhmrvdcocidgtuxasdqkwsdvijmnpmayhfiva"; CheckLCSLength(s1, s2, 288); } } diff --git a/library/cpp/lcs/ya.make b/library/cpp/lcs/ya.make index 3a7caafc88..9b49f3a061 100644 --- a/library/cpp/lcs/ya.make +++ b/library/cpp/lcs/ya.make @@ -1,11 +1,11 @@ LIBRARY() -OWNER(velavokr) - -PEERDIR( +OWNER(velavokr) + +PEERDIR( library/cpp/containers/paged_vector -) - +) + SRCS( lcs_via_lis.cpp ) diff --git a/library/cpp/lfalloc/dbg_info/dbg_info.cpp b/library/cpp/lfalloc/dbg_info/dbg_info.cpp index 1fb9f7ad93..efdd70a49f 100644 --- a/library/cpp/lfalloc/dbg_info/dbg_info.cpp +++ b/library/cpp/lfalloc/dbg_info/dbg_info.cpp @@ -3,122 +3,122 @@ #include <library/cpp/malloc/api/malloc.h> namespace NAllocDbg { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - using TGetAllocationCounter = i64(int counter); + using TGetAllocationCounter = i64(int counter); - using TSetThreadAllocTag = int(int tag); - using TGetPerTagAllocInfo = void( - bool flushPerThreadCounters, - TPerTagAllocInfo* info, - int& maxTag, - int& numSizes); + using TSetThreadAllocTag = int(int tag); + using TGetPerTagAllocInfo = void( + bool flushPerThreadCounters, + TPerTagAllocInfo* info, + int& maxTag, + int& numSizes); - using TSetProfileCurrentThread = bool(bool newVal); + using TSetProfileCurrentThread = bool(bool newVal); using TSetProfileAllThreads = bool(bool newVal); - using TSetAllocationSamplingEnabled = bool(bool newVal); + using TSetAllocationSamplingEnabled = bool(bool newVal); - using TSetAllocationSampleRate = size_t(size_t newVal); - using TSetAllocationSampleMaxSize = size_t(size_t newVal); + using TSetAllocationSampleRate = size_t(size_t newVal); + using TSetAllocationSampleMaxSize = size_t(size_t newVal); - using TSetAllocationCallback = TAllocationCallback*(TAllocationCallback* newVal); - using TSetDeallocationCallback = TDeallocationCallback*(TDeallocationCallback* newVal); + using TSetAllocationCallback = TAllocationCallback*(TAllocationCallback* newVal); + using TSetDeallocationCallback = TDeallocationCallback*(TDeallocationCallback* newVal); - struct TAllocFn { - TGetAllocationCounter* GetAllocationCounterFast = nullptr; - TGetAllocationCounter* GetAllocationCounterFull = nullptr; + struct TAllocFn { + TGetAllocationCounter* GetAllocationCounterFast = nullptr; + TGetAllocationCounter* GetAllocationCounterFull = nullptr; - TSetThreadAllocTag* SetThreadAllocTag = nullptr; - TGetPerTagAllocInfo* GetPerTagAllocInfo = nullptr; + TSetThreadAllocTag* SetThreadAllocTag = nullptr; + TGetPerTagAllocInfo* GetPerTagAllocInfo = nullptr; - TSetProfileCurrentThread* SetProfileCurrentThread = nullptr; + TSetProfileCurrentThread* SetProfileCurrentThread = nullptr; TSetProfileAllThreads* SetProfileAllThreads = nullptr; - TSetAllocationSamplingEnabled* SetAllocationSamplingEnabled = nullptr; + TSetAllocationSamplingEnabled* SetAllocationSamplingEnabled = nullptr; - TSetAllocationSampleRate* SetAllocationSampleRate = nullptr; - TSetAllocationSampleMaxSize* SetAllocationSampleMaxSize = nullptr; + TSetAllocationSampleRate* SetAllocationSampleRate = nullptr; + TSetAllocationSampleMaxSize* SetAllocationSampleMaxSize = nullptr; - TSetAllocationCallback* SetAllocationCallback = nullptr; - TSetDeallocationCallback* SetDeallocationCallback = nullptr; + TSetAllocationCallback* SetAllocationCallback = nullptr; + TSetDeallocationCallback* SetDeallocationCallback = nullptr; - TAllocFn() { - auto mallocInfo = NMalloc::MallocInfo(); + TAllocFn() { + auto mallocInfo = NMalloc::MallocInfo(); - GetAllocationCounterFast = (TGetAllocationCounter*)mallocInfo.GetParam("GetLFAllocCounterFast"); - GetAllocationCounterFull = (TGetAllocationCounter*)mallocInfo.GetParam("GetLFAllocCounterFull"); + GetAllocationCounterFast = (TGetAllocationCounter*)mallocInfo.GetParam("GetLFAllocCounterFast"); + GetAllocationCounterFull = (TGetAllocationCounter*)mallocInfo.GetParam("GetLFAllocCounterFull"); - SetThreadAllocTag = (TSetThreadAllocTag*)mallocInfo.GetParam("SetThreadAllocTag"); - GetPerTagAllocInfo = (TGetPerTagAllocInfo*)mallocInfo.GetParam("GetPerTagAllocInfo"); + SetThreadAllocTag = (TSetThreadAllocTag*)mallocInfo.GetParam("SetThreadAllocTag"); + GetPerTagAllocInfo = (TGetPerTagAllocInfo*)mallocInfo.GetParam("GetPerTagAllocInfo"); - SetProfileCurrentThread = (TSetProfileCurrentThread*)mallocInfo.GetParam("SetProfileCurrentThread"); + SetProfileCurrentThread = (TSetProfileCurrentThread*)mallocInfo.GetParam("SetProfileCurrentThread"); SetProfileAllThreads = (TSetProfileAllThreads*)mallocInfo.GetParam("SetProfileAllThreads"); - SetAllocationSamplingEnabled = (TSetAllocationSamplingEnabled*)mallocInfo.GetParam("SetAllocationSamplingEnabled"); - - SetAllocationSampleRate = (TSetAllocationSampleRate*)mallocInfo.GetParam("SetAllocationSampleRate"); - SetAllocationSampleMaxSize = (TSetAllocationSampleMaxSize*)mallocInfo.GetParam("SetAllocationSampleMaxSize"); - - SetAllocationCallback = (TSetAllocationCallback*)mallocInfo.GetParam("SetAllocationCallback"); - SetDeallocationCallback = (TSetDeallocationCallback*)mallocInfo.GetParam("SetDeallocationCallback"); - } - }; - - //////////////////////////////////////////////////////////////////////////////// - - static TAllocFn AllocFn; - - i64 GetAllocationCounterFast(ELFAllocCounter counter) { - return AllocFn.GetAllocationCounterFast ? AllocFn.GetAllocationCounterFast(counter) : 0; - } - - i64 GetAllocationCounterFull(ELFAllocCounter counter) { - return AllocFn.GetAllocationCounterFull ? AllocFn.GetAllocationCounterFull(counter) : 0; - } - - int SetThreadAllocTag(int tag) { - return AllocFn.SetThreadAllocTag ? AllocFn.SetThreadAllocTag(tag) : 0; - } - - TArrayPtr<TPerTagAllocInfo> GetPerTagAllocInfo( - bool flushPerThreadCounters, - int& maxTag, - int& numSizes) { - if (AllocFn.GetPerTagAllocInfo) { - AllocFn.GetPerTagAllocInfo(flushPerThreadCounters, nullptr, maxTag, numSizes); - TArrayPtr<TPerTagAllocInfo> info = new TPerTagAllocInfo[maxTag * numSizes]; - AllocFn.GetPerTagAllocInfo(flushPerThreadCounters, info.Get(), maxTag, numSizes); - return info; - } - maxTag = 0; - numSizes = 0; - return nullptr; - } - - bool SetProfileCurrentThread(bool newVal) { - return AllocFn.SetProfileCurrentThread ? AllocFn.SetProfileCurrentThread(newVal) : false; + SetAllocationSamplingEnabled = (TSetAllocationSamplingEnabled*)mallocInfo.GetParam("SetAllocationSamplingEnabled"); + + SetAllocationSampleRate = (TSetAllocationSampleRate*)mallocInfo.GetParam("SetAllocationSampleRate"); + SetAllocationSampleMaxSize = (TSetAllocationSampleMaxSize*)mallocInfo.GetParam("SetAllocationSampleMaxSize"); + + SetAllocationCallback = (TSetAllocationCallback*)mallocInfo.GetParam("SetAllocationCallback"); + SetDeallocationCallback = (TSetDeallocationCallback*)mallocInfo.GetParam("SetDeallocationCallback"); + } + }; + + //////////////////////////////////////////////////////////////////////////////// + + static TAllocFn AllocFn; + + i64 GetAllocationCounterFast(ELFAllocCounter counter) { + return AllocFn.GetAllocationCounterFast ? AllocFn.GetAllocationCounterFast(counter) : 0; + } + + i64 GetAllocationCounterFull(ELFAllocCounter counter) { + return AllocFn.GetAllocationCounterFull ? AllocFn.GetAllocationCounterFull(counter) : 0; + } + + int SetThreadAllocTag(int tag) { + return AllocFn.SetThreadAllocTag ? AllocFn.SetThreadAllocTag(tag) : 0; + } + + TArrayPtr<TPerTagAllocInfo> GetPerTagAllocInfo( + bool flushPerThreadCounters, + int& maxTag, + int& numSizes) { + if (AllocFn.GetPerTagAllocInfo) { + AllocFn.GetPerTagAllocInfo(flushPerThreadCounters, nullptr, maxTag, numSizes); + TArrayPtr<TPerTagAllocInfo> info = new TPerTagAllocInfo[maxTag * numSizes]; + AllocFn.GetPerTagAllocInfo(flushPerThreadCounters, info.Get(), maxTag, numSizes); + return info; + } + maxTag = 0; + numSizes = 0; + return nullptr; + } + + bool SetProfileCurrentThread(bool newVal) { + return AllocFn.SetProfileCurrentThread ? AllocFn.SetProfileCurrentThread(newVal) : false; } bool SetProfileAllThreads(bool newVal) { return AllocFn.SetProfileAllThreads ? AllocFn.SetProfileAllThreads(newVal) : false; } - bool SetAllocationSamplingEnabled(bool newVal) { - return AllocFn.SetAllocationSamplingEnabled ? AllocFn.SetAllocationSamplingEnabled(newVal) : false; - } + bool SetAllocationSamplingEnabled(bool newVal) { + return AllocFn.SetAllocationSamplingEnabled ? AllocFn.SetAllocationSamplingEnabled(newVal) : false; + } - size_t SetAllocationSampleRate(size_t newVal) { - return AllocFn.SetAllocationSampleRate ? AllocFn.SetAllocationSampleRate(newVal) : 0; - } + size_t SetAllocationSampleRate(size_t newVal) { + return AllocFn.SetAllocationSampleRate ? AllocFn.SetAllocationSampleRate(newVal) : 0; + } - size_t SetAllocationSampleMaxSize(size_t newVal) { - return AllocFn.SetAllocationSampleMaxSize ? AllocFn.SetAllocationSampleMaxSize(newVal) : 0; - } + size_t SetAllocationSampleMaxSize(size_t newVal) { + return AllocFn.SetAllocationSampleMaxSize ? AllocFn.SetAllocationSampleMaxSize(newVal) : 0; + } - TAllocationCallback* SetAllocationCallback(TAllocationCallback* newVal) { - return AllocFn.SetAllocationCallback ? AllocFn.SetAllocationCallback(newVal) : nullptr; - } + TAllocationCallback* SetAllocationCallback(TAllocationCallback* newVal) { + return AllocFn.SetAllocationCallback ? AllocFn.SetAllocationCallback(newVal) : nullptr; + } - TDeallocationCallback* SetDeallocationCallback(TDeallocationCallback* newVal) { - return AllocFn.SetDeallocationCallback ? AllocFn.SetDeallocationCallback(newVal) : nullptr; - } + TDeallocationCallback* SetDeallocationCallback(TDeallocationCallback* newVal) { + return AllocFn.SetDeallocationCallback ? AllocFn.SetDeallocationCallback(newVal) : nullptr; + } } diff --git a/library/cpp/lfalloc/dbg_info/dbg_info.h b/library/cpp/lfalloc/dbg_info/dbg_info.h index 071562a81a..0e9bbad869 100644 --- a/library/cpp/lfalloc/dbg_info/dbg_info.h +++ b/library/cpp/lfalloc/dbg_info/dbg_info.h @@ -4,74 +4,74 @@ #include <util/system/types.h> namespace NAllocDbg { - //////////////////////////////////////////////////////////////////////////////// - // Allocation statistics - - enum ELFAllocCounter { - CT_USER_ALLOC, // accumulated size requested by user code - CT_MMAP, // accumulated mmapped size - CT_MMAP_CNT, // number of mmapped regions - CT_MUNMAP, // accumulated unmmapped size - CT_MUNMAP_CNT, // number of munmaped regions - CT_SYSTEM_ALLOC, // accumulated allocated size for internal lfalloc needs - CT_SYSTEM_FREE, // accumulated deallocated size for internal lfalloc needs - CT_SMALL_ALLOC, // accumulated allocated size for fixed-size blocks - CT_SMALL_FREE, // accumulated deallocated size for fixed-size blocks - CT_LARGE_ALLOC, // accumulated allocated size for large blocks - CT_LARGE_FREE, // accumulated deallocated size for large blocks - CT_SLOW_ALLOC_CNT, // number of slow (not LF) allocations - CT_DEGRAGMENT_CNT, // number of memory defragmentations - CT_MAX - }; - - i64 GetAllocationCounterFast(ELFAllocCounter counter); - i64 GetAllocationCounterFull(ELFAllocCounter counter); - - //////////////////////////////////////////////////////////////////////////////// - // Allocation statistics could be tracked on per-tag basis - - int SetThreadAllocTag(int tag); - - class TScopedTag { - private: - int PrevTag; - - public: - explicit TScopedTag(int tag) { - PrevTag = SetThreadAllocTag(tag); - } - - ~TScopedTag() { - SetThreadAllocTag(PrevTag); - } - }; - - struct TPerTagAllocInfo { - ssize_t Count; - ssize_t Size; - }; - - TArrayPtr<TPerTagAllocInfo> GetPerTagAllocInfo( - bool flushPerThreadCounters, - int& maxTag, - int& numSizes); - - //////////////////////////////////////////////////////////////////////////////// - // Allocation sampling could be used to collect detailed information - - bool SetProfileCurrentThread(bool newVal); + //////////////////////////////////////////////////////////////////////////////// + // Allocation statistics + + enum ELFAllocCounter { + CT_USER_ALLOC, // accumulated size requested by user code + CT_MMAP, // accumulated mmapped size + CT_MMAP_CNT, // number of mmapped regions + CT_MUNMAP, // accumulated unmmapped size + CT_MUNMAP_CNT, // number of munmaped regions + CT_SYSTEM_ALLOC, // accumulated allocated size for internal lfalloc needs + CT_SYSTEM_FREE, // accumulated deallocated size for internal lfalloc needs + CT_SMALL_ALLOC, // accumulated allocated size for fixed-size blocks + CT_SMALL_FREE, // accumulated deallocated size for fixed-size blocks + CT_LARGE_ALLOC, // accumulated allocated size for large blocks + CT_LARGE_FREE, // accumulated deallocated size for large blocks + CT_SLOW_ALLOC_CNT, // number of slow (not LF) allocations + CT_DEGRAGMENT_CNT, // number of memory defragmentations + CT_MAX + }; + + i64 GetAllocationCounterFast(ELFAllocCounter counter); + i64 GetAllocationCounterFull(ELFAllocCounter counter); + + //////////////////////////////////////////////////////////////////////////////// + // Allocation statistics could be tracked on per-tag basis + + int SetThreadAllocTag(int tag); + + class TScopedTag { + private: + int PrevTag; + + public: + explicit TScopedTag(int tag) { + PrevTag = SetThreadAllocTag(tag); + } + + ~TScopedTag() { + SetThreadAllocTag(PrevTag); + } + }; + + struct TPerTagAllocInfo { + ssize_t Count; + ssize_t Size; + }; + + TArrayPtr<TPerTagAllocInfo> GetPerTagAllocInfo( + bool flushPerThreadCounters, + int& maxTag, + int& numSizes); + + //////////////////////////////////////////////////////////////////////////////// + // Allocation sampling could be used to collect detailed information + + bool SetProfileCurrentThread(bool newVal); bool SetProfileAllThreads(bool newVal); - bool SetAllocationSamplingEnabled(bool newVal); + bool SetAllocationSamplingEnabled(bool newVal); - size_t SetAllocationSampleRate(size_t newVal); - size_t SetAllocationSampleMaxSize(size_t newVal); + size_t SetAllocationSampleRate(size_t newVal); + size_t SetAllocationSampleMaxSize(size_t newVal); #define DBG_ALLOC_INVALID_COOKIE (-1) - using TAllocationCallback = int(int tag, size_t size, int sizeIdx); - using TDeallocationCallback = void(int cookie, int tag, size_t size, int sizeIdx); + using TAllocationCallback = int(int tag, size_t size, int sizeIdx); + using TDeallocationCallback = void(int cookie, int tag, size_t size, int sizeIdx); - TAllocationCallback* SetAllocationCallback(TAllocationCallback* newVal); - TDeallocationCallback* SetDeallocationCallback(TDeallocationCallback* newVal); + TAllocationCallback* SetAllocationCallback(TAllocationCallback* newVal); + TDeallocationCallback* SetDeallocationCallback(TDeallocationCallback* newVal); -} +} diff --git a/library/cpp/lfalloc/lf_allocX64.cpp b/library/cpp/lfalloc/lf_allocX64.cpp index 2eb90761fe..5d642e79c8 100644 --- a/library/cpp/lfalloc/lf_allocX64.cpp +++ b/library/cpp/lfalloc/lf_allocX64.cpp @@ -9,7 +9,7 @@ #endif #ifndef _darwin_ -#if !defined(YMAKE) +#if !defined(YMAKE) void* operator new(size_t size) { return LFAlloc(size); } @@ -18,11 +18,11 @@ void* operator new(size_t size, const std::nothrow_t&) OP_THROWNOTHING { return LFAlloc(size); } -void operator delete(void* p)OP_THROWNOTHING { +void operator delete(void* p)OP_THROWNOTHING { LFFree(p); } -void operator delete(void* p, const std::nothrow_t&)OP_THROWNOTHING { +void operator delete(void* p, const std::nothrow_t&)OP_THROWNOTHING { LFFree(p); } @@ -41,7 +41,7 @@ void operator delete[](void* p) OP_THROWNOTHING { void operator delete[](void* p, const std::nothrow_t&) OP_THROWNOTHING { LFFree(p); } -#endif +#endif //#ifndef _MSC_VER @@ -53,21 +53,21 @@ extern "C" void* valloc(size_t size) { return LFVAlloc(size); } -extern "C" int posix_memalign(void** memptr, size_t alignment, size_t size) { +extern "C" int posix_memalign(void** memptr, size_t alignment, size_t size) { return LFPosixMemalign(memptr, alignment, size); } extern "C" void* memalign(size_t alignment, size_t size) { void* ptr; int res = LFPosixMemalign(&ptr, alignment, size); - return res ? nullptr : ptr; + return res ? nullptr : ptr; } extern "C" void* aligned_alloc(size_t alignment, size_t size) { return memalign(alignment, size); } -#if !defined(_MSC_VER) && !defined(_freebsd_) +#if !defined(_MSC_VER) && !defined(_freebsd_) // Workaround for pthread_create bug in linux. extern "C" void* __libc_memalign(size_t alignment, size_t size) { return memalign(alignment, size); @@ -81,8 +81,8 @@ extern "C" void free(void* ptr) { extern "C" void* calloc(size_t n, size_t elem_size) { // Overflow check const size_t size = n * elem_size; - if (elem_size != 0 && size / elem_size != n) - return nullptr; + if (elem_size != 0 && size / elem_size != n) + return nullptr; void* result = LFAlloc(size); if (result != nullptr) { diff --git a/library/cpp/lfalloc/lf_allocX64.h b/library/cpp/lfalloc/lf_allocX64.h index fd2a906d6f..d0c3ada388 100644 --- a/library/cpp/lfalloc/lf_allocX64.h +++ b/library/cpp/lfalloc/lf_allocX64.h @@ -12,13 +12,13 @@ #ifdef _MSC_VER #ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_WARNINGS #endif #ifdef _M_X64 -#define _64_ +#define _64_ #endif #include <intrin.h> -#define WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN #include <Windows.h> #pragma intrinsic(_InterlockedCompareExchange) #pragma intrinsic(_InterlockedExchangeAdd) @@ -33,7 +33,7 @@ using TAtomic = volatile long; -static inline long AtomicAdd(TAtomic& a, long b) { +static inline long AtomicAdd(TAtomic& a, long b) { return _InterlockedExchangeAdd(&a, b) + b; } @@ -44,14 +44,14 @@ static inline long AtomicSub(TAtomic& a, long b) { #pragma comment(lib, "synchronization.lib") #ifndef NDEBUG -#define Y_ASSERT_NOBT(x) \ - { \ - if (IsDebuggerPresent()) { \ - if (!(x)) \ - __debugbreak(); \ - } else \ - assert(x); \ - } +#define Y_ASSERT_NOBT(x) \ + { \ + if (IsDebuggerPresent()) { \ + if (!(x)) \ + __debugbreak(); \ + } else \ + assert(x); \ + } #else #define Y_ASSERT_NOBT(x) ((void)0) #endif @@ -101,13 +101,13 @@ static inline long AtomicSub(TAtomic& a, long b) { #if defined(_linux_) #include <linux/futex.h> #include <sys/syscall.h> -#if !defined(MADV_HUGEPAGE) -#define MADV_HUGEPAGE 14 -#endif -#if !defined(MAP_HUGETLB) -#define MAP_HUGETLB 0x40000 -#endif +#if !defined(MADV_HUGEPAGE) +#define MADV_HUGEPAGE 14 #endif +#if !defined(MAP_HUGETLB) +#define MAP_HUGETLB 0x40000 +#endif +#endif #define PERTHREAD __thread @@ -124,8 +124,8 @@ static inline long AtomicSub(TAtomic& a, long b) { static bool FillMemoryOnAllocation = true; #endif -static bool TransparentHugePages = false; // force MADV_HUGEPAGE for large allocs -static bool MapHugeTLB = false; // force MAP_HUGETLB for small allocs +static bool TransparentHugePages = false; // force MADV_HUGEPAGE for large allocs +static bool MapHugeTLB = false; // force MAP_HUGETLB for small allocs static bool EnableDefrag = true; // Buffers that are larger than this size will not be filled with 0xcf @@ -133,8 +133,8 @@ static bool EnableDefrag = true; #define DBG_FILL_MAX_SIZE 0x01000000000000ULL #endif -template <class T> -inline T* DoCas(T* volatile* target, T* exchange, T* compare) { +template <class T> +inline T* DoCas(T* volatile* target, T* exchange, T* compare) { #if defined(__has_builtin) && __has_builtin(__sync_val_compare_and_swap) return __sync_val_compare_and_swap(target, compare, exchange); #elif defined(_WIN32) @@ -145,22 +145,22 @@ inline T* DoCas(T* volatile* target, T* exchange, T* compare) { return (T*)_InterlockedCompareExchange((LONG*)target, (LONG)exchange, (LONG)compare); #endif #elif defined(__i386) || defined(__x86_64__) - union { - T* volatile* NP; - void* volatile* VoidP; + union { + T* volatile* NP; + void* volatile* VoidP; } gccSucks; gccSucks.NP = target; - void* volatile* targetVoidP = gccSucks.VoidP; + void* volatile* targetVoidP = gccSucks.VoidP; - __asm__ __volatile__( + __asm__ __volatile__( "lock\n\t" "cmpxchg %2,%0\n\t" - : "+m"(*(targetVoidP)), "+a"(compare) - : "r"(exchange) + : "+m"(*(targetVoidP)), "+a"(compare) + : "r"(exchange) : "cc", "memory"); return compare; #else -#error inline_cas not defined for this platform +#error inline_cas not defined for this platform #endif } @@ -192,14 +192,14 @@ const int nSizeIdxToSize[N_SIZES] = { #if defined(_64_) 16, 16, 32, 32, 48, 64, 96, 128, #else - 8, - 16, - 24, - 32, - 48, - 64, - 96, - 128, + 8, + 16, + 24, + 32, + 48, + 64, + 96, + 128, #endif 192, 256, 384, 512, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192, 12288, 16384, 24576, 32768, @@ -215,40 +215,40 @@ const size_t N_MAX_FAST_SIZE = 32768; const unsigned char size2idxArr1[64 + 1] = { 1, #if defined(_64_) - 2, 2, 4, 4, // 16, 16, 32, 32 + 2, 2, 4, 4, // 16, 16, 32, 32 #else - 1, 2, 3, 4, // 8, 16, 24, 32 + 1, 2, 3, 4, // 8, 16, 24, 32 #endif - 5, 5, 6, 6, // 48, 64 - 7, 7, 7, 7, 8, 8, 8, 8, // 96, 128 - 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, // 192, 256 - 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, // 384 - 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12 // 512 + 5, 5, 6, 6, // 48, 64 + 7, 7, 7, 7, 8, 8, 8, 8, // 96, 128 + 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, // 192, 256 + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, // 384 + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12 // 512 }; #ifdef LFALLOC_YT const unsigned char size2idxArr2[256] = { #else const unsigned char size2idxArr2[128] = { #endif - 12, 12, 13, 14, // 512, 512, 768, 1024 - 15, 15, 16, 16, // 1536, 2048 - 17, 17, 17, 17, 18, 18, 18, 18, // 3072, 4096 - 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, // 6144, 8192 - 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, // 12288 - 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, // 16384 - 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, - 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, // 24576 - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, - 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, // 32768 + 12, 12, 13, 14, // 512, 512, 768, 1024 + 15, 15, 16, 16, // 1536, 2048 + 17, 17, 17, 17, 18, 18, 18, 18, // 3072, 4096 + 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, // 6144, 8192 + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, // 12288 + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, // 16384 + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, // 24576 + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, // 32768 #ifdef LFALLOC_YT - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, - 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, // 49152 - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, - 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, // 65536 + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, // 49152 + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, + 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, // 65536 #endif }; @@ -259,14 +259,14 @@ const int FREE_CHUNK_ARR_BUF = 0x20000; // this is effectively 128G of free memo static volatile uintptr_t freeChunkArr[FREE_CHUNK_ARR_BUF]; static volatile int freeChunkCount; -static void AddFreeChunk(uintptr_t chunkId) { +static void AddFreeChunk(uintptr_t chunkId) { chunkSizeIdx[chunkId] = -1; if (Y_UNLIKELY(freeChunkCount == FREE_CHUNK_ARR_BUF)) NMalloc::AbortFromCorruptedAllocator("free chunks array overflowed"); freeChunkArr[freeChunkCount++] = chunkId; } -static bool GetFreeChunk(uintptr_t* res) { +static bool GetFreeChunk(uintptr_t* res) { if (freeChunkCount == 0) { *res = 0; return false; @@ -276,29 +276,29 @@ static bool GetFreeChunk(uintptr_t* res) { } ////////////////////////////////////////////////////////////////////////// -enum ELFAllocCounter { - CT_USER_ALLOC, // accumulated size requested by user code - CT_MMAP, // accumulated mmapped size - CT_MMAP_CNT, // number of mmapped regions - CT_MUNMAP, // accumulated unmmapped size - CT_MUNMAP_CNT, // number of munmaped regions - CT_SYSTEM_ALLOC, // accumulated allocated size for internal lfalloc needs - CT_SYSTEM_FREE, // accumulated deallocated size for internal lfalloc needs - CT_SMALL_ALLOC, // accumulated allocated size for fixed-size blocks - CT_SMALL_FREE, // accumulated deallocated size for fixed-size blocks - CT_LARGE_ALLOC, // accumulated allocated size for large blocks - CT_LARGE_FREE, // accumulated deallocated size for large blocks - CT_SLOW_ALLOC_CNT, // number of slow (not LF) allocations - CT_DEGRAGMENT_CNT, // number of memory defragmentations +enum ELFAllocCounter { + CT_USER_ALLOC, // accumulated size requested by user code + CT_MMAP, // accumulated mmapped size + CT_MMAP_CNT, // number of mmapped regions + CT_MUNMAP, // accumulated unmmapped size + CT_MUNMAP_CNT, // number of munmaped regions + CT_SYSTEM_ALLOC, // accumulated allocated size for internal lfalloc needs + CT_SYSTEM_FREE, // accumulated deallocated size for internal lfalloc needs + CT_SMALL_ALLOC, // accumulated allocated size for fixed-size blocks + CT_SMALL_FREE, // accumulated deallocated size for fixed-size blocks + CT_LARGE_ALLOC, // accumulated allocated size for large blocks + CT_LARGE_FREE, // accumulated deallocated size for large blocks + CT_SLOW_ALLOC_CNT, // number of slow (not LF) allocations + CT_DEGRAGMENT_CNT, // number of memory defragmentations CT_MAX }; static Y_FORCE_INLINE void IncrementCounter(ELFAllocCounter counter, size_t value); ////////////////////////////////////////////////////////////////////////// -enum EMMapMode { - MM_NORMAL, // memory for small allocs - MM_HUGE // memory for large allocs +enum EMMapMode { + MM_NORMAL, // memory for small allocs + MM_HUGE // memory for large allocs }; #ifndef _MSC_VER @@ -308,14 +308,14 @@ inline void VerifyMmapResult(void* result) { } #endif -#if !defined(_MSC_VER) && !defined(_freebsd_) && defined(_64_) -static char* AllocWithMMapLinuxImpl(uintptr_t sz, EMMapMode mode) { - char* volatile* areaPtr; - char* areaStart; +#if !defined(_MSC_VER) && !defined(_freebsd_) && defined(_64_) +static char* AllocWithMMapLinuxImpl(uintptr_t sz, EMMapMode mode) { + char* volatile* areaPtr; + char* areaStart; uintptr_t areaFinish; - int mapProt = PROT_READ | PROT_WRITE; - int mapFlags = MAP_PRIVATE | MAP_ANON; + int mapProt = PROT_READ | PROT_WRITE; + int mapFlags = MAP_PRIVATE | MAP_ANON; if (mode == MM_HUGE) { areaPtr = reinterpret_cast<char* volatile*>(&linuxAllocPointerHuge); @@ -332,9 +332,9 @@ static char* AllocWithMMapLinuxImpl(uintptr_t sz, EMMapMode mode) { } bool wrapped = false; - for (;;) { - char* prevAllocPtr = *areaPtr; - char* nextAllocPtr = prevAllocPtr + sz; + for (;;) { + char* prevAllocPtr = *areaPtr; + char* nextAllocPtr = prevAllocPtr + sz; if (uintptr_t(nextAllocPtr - (char*)nullptr) >= areaFinish) { if (Y_UNLIKELY(wrapped)) { NMalloc::AbortFromCorruptedAllocator("virtual memory is over fragmented"); @@ -348,7 +348,7 @@ static char* AllocWithMMapLinuxImpl(uintptr_t sz, EMMapMode mode) { if (DoCas(areaPtr, nextAllocPtr, prevAllocPtr) != prevAllocPtr) continue; - char* largeBlock = (char*)mmap(prevAllocPtr, sz, mapProt, mapFlags, -1, 0); + char* largeBlock = (char*)mmap(prevAllocPtr, sz, mapProt, mapFlags, -1, 0); VerifyMmapResult(largeBlock); if (largeBlock == prevAllocPtr) return largeBlock; @@ -363,22 +363,22 @@ static char* AllocWithMMapLinuxImpl(uintptr_t sz, EMMapMode mode) { } #endif -static char* AllocWithMMap(uintptr_t sz, EMMapMode mode) { +static char* AllocWithMMap(uintptr_t sz, EMMapMode mode) { (void)mode; #ifdef _MSC_VER - char* largeBlock = (char*)VirtualAlloc(0, sz, MEM_RESERVE, PAGE_READWRITE); + char* largeBlock = (char*)VirtualAlloc(0, sz, MEM_RESERVE, PAGE_READWRITE); if (Y_UNLIKELY(largeBlock == nullptr)) NMalloc::AbortFromCorruptedAllocator("out of memory"); if (Y_UNLIKELY(uintptr_t(((char*)largeBlock - ALLOC_START) + sz) >= N_MAX_WORKSET_SIZE)) NMalloc::AbortFromCorruptedAllocator("out of working set, something has broken"); #else -#if defined(_freebsd_) || !defined(_64_) - char* largeBlock = (char*)mmap(0, sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); +#if defined(_freebsd_) || !defined(_64_) + char* largeBlock = (char*)mmap(0, sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); VerifyMmapResult(largeBlock); if (Y_UNLIKELY(uintptr_t(((char*)largeBlock - ALLOC_START) + sz) >= N_MAX_WORKSET_SIZE)) NMalloc::AbortFromCorruptedAllocator("out of working set, something has broken"); #else - char* largeBlock = AllocWithMMapLinuxImpl(sz, mode); + char* largeBlock = AllocWithMMapLinuxImpl(sz, mode); if (TransparentHugePages) { madvise(largeBlock, sz, MADV_HUGEPAGE); } @@ -444,14 +444,14 @@ static void LargeBlockUnmap(void* p, size_t pages) { const size_t LB_BUF_SIZE = 250; const size_t LB_BUF_HASH = 977; static int LB_LIMIT_TOTAL_SIZE = 500 * 1024 * 1024 / 4096; // do not keep more then this mem total in lbFreePtrs[] -static void* volatile lbFreePtrs[LB_BUF_HASH][LB_BUF_SIZE]; +static void* volatile lbFreePtrs[LB_BUF_HASH][LB_BUF_SIZE]; static TAtomic lbFreePageCount; -static void* LargeBlockAlloc(size_t _nSize, ELFAllocCounter counter) { +static void* LargeBlockAlloc(size_t _nSize, ELFAllocCounter counter) { size_t pgCount = (_nSize + 4095) / 4096; #ifdef _MSC_VER - char* pRes = (char*)VirtualAlloc(0, (pgCount + 1) * 4096ll, MEM_COMMIT, PAGE_READWRITE); + char* pRes = (char*)VirtualAlloc(0, (pgCount + 1) * 4096ll, MEM_COMMIT, PAGE_READWRITE); if (Y_UNLIKELY(pRes == 0)) { NMalloc::AbortFromCorruptedAllocator("out of memory"); } @@ -462,7 +462,7 @@ static void* LargeBlockAlloc(size_t _nSize, ELFAllocCounter counter) { int lbHash = pgCount % LB_BUF_HASH; for (int i = 0; i < LB_BUF_SIZE; ++i) { - void* p = lbFreePtrs[lbHash][i]; + void* p = lbFreePtrs[lbHash][i]; if (p == nullptr) continue; if (DoCas(&lbFreePtrs[lbHash][i], (void*)nullptr, p) == p) { @@ -474,14 +474,14 @@ static void* LargeBlockAlloc(size_t _nSize, ELFAllocCounter counter) { } else { if (DoCas(&lbFreePtrs[lbHash][i], p, (void*)nullptr) != (void*)nullptr) { // block was freed while we were busy - AtomicAdd(lbFreePageCount, -realPageCount); + AtomicAdd(lbFreePageCount, -realPageCount); LargeBlockUnmap(p, realPageCount); --i; } } } } - char* pRes = AllocWithMMap((pgCount + 1) * 4096ll, MM_HUGE); + char* pRes = AllocWithMMap((pgCount + 1) * 4096ll, MM_HUGE); #endif pRes += 4096ll; TLargeBlk::As(pRes)->SetSize(_nSize, pgCount); @@ -491,7 +491,7 @@ static void* LargeBlockAlloc(size_t _nSize, ELFAllocCounter counter) { } #ifndef _MSC_VER -static void FreeAllLargeBlockMem() { +static void FreeAllLargeBlockMem() { for (auto& lbFreePtr : lbFreePtrs) { for (int i = 0; i < LB_BUF_SIZE; ++i) { void* p = lbFreePtr[i]; @@ -507,7 +507,7 @@ static void FreeAllLargeBlockMem() { } #endif -static void LargeBlockFree(void* p, ELFAllocCounter counter) { +static void LargeBlockFree(void* p, ELFAllocCounter counter) { if (p == nullptr) return; #ifdef _MSC_VER @@ -525,7 +525,7 @@ static void LargeBlockFree(void* p, ELFAllocCounter counter) { for (int i = 0; i < LB_BUF_SIZE; ++i) { if (lbFreePtrs[lbHash][i] == nullptr) { if (DoCas(&lbFreePtrs[lbHash][i], p, (void*)nullptr) == nullptr) { - AtomicAdd(lbFreePageCount, pgCount); + AtomicAdd(lbFreePageCount, pgCount); return; } } @@ -535,11 +535,11 @@ static void LargeBlockFree(void* p, ELFAllocCounter counter) { #endif } -static void* SystemAlloc(size_t _nSize) { +static void* SystemAlloc(size_t _nSize) { //HeapAlloc(GetProcessHeap(), HEAP_GENERATE_EXCEPTIONS, _nSize); return LargeBlockAlloc(_nSize, CT_SYSTEM_ALLOC); } -static void SystemFree(void* p) { +static void SystemFree(void* p) { //HeapFree(GetProcessHeap(), 0, p); LargeBlockFree(p, CT_SYSTEM_FREE); } @@ -659,29 +659,29 @@ public: }; ////////////////////////////////////////////////////////////////////////// -class TLFAllocFreeList { - struct TNode { - TNode* Next; +class TLFAllocFreeList { + struct TNode { + TNode* Next; }; - TNode* volatile Head; - TNode* volatile Pending; + TNode* volatile Head; + TNode* volatile Pending; TAtomic PendingToFreeListCounter; TAtomic AllocCount; - void* Padding; + void* Padding; - static Y_FORCE_INLINE void Enqueue(TNode* volatile* headPtr, TNode* n) { - for (;;) { - TNode* volatile prevHead = *headPtr; + static Y_FORCE_INLINE void Enqueue(TNode* volatile* headPtr, TNode* n) { + for (;;) { + TNode* volatile prevHead = *headPtr; n->Next = prevHead; if (DoCas(headPtr, n, prevHead) == prevHead) break; } } - Y_FORCE_INLINE void* DoAlloc() { - TNode* res; + Y_FORCE_INLINE void* DoAlloc() { + TNode* res; for (res = Head; res; res = Head) { - TNode* keepNext = res->Next; + TNode* keepNext = res->Next; if (DoCas(&Head, keepNext, res) == res) { //Y_VERIFY(keepNext == res->Next); break; @@ -689,63 +689,63 @@ class TLFAllocFreeList { } return res; } - void FreeList(TNode* fl) { + void FreeList(TNode* fl) { if (!fl) return; - TNode* flTail = fl; + TNode* flTail = fl; while (flTail->Next) flTail = flTail->Next; - for (;;) { - TNode* volatile prevHead = Head; + for (;;) { + TNode* volatile prevHead = Head; flTail->Next = prevHead; if (DoCas(&Head, fl, prevHead) == prevHead) break; } } - + public: - Y_FORCE_INLINE void Free(void* ptr) { - TNode* newFree = (TNode*)ptr; - if (AtomicAdd(AllocCount, 0) == 0) + Y_FORCE_INLINE void Free(void* ptr) { + TNode* newFree = (TNode*)ptr; + if (AtomicAdd(AllocCount, 0) == 0) Enqueue(&Head, newFree); else Enqueue(&Pending, newFree); } - Y_FORCE_INLINE void* Alloc() { + Y_FORCE_INLINE void* Alloc() { TAtomic keepCounter = AtomicAdd(PendingToFreeListCounter, 0); - TNode* fl = Pending; - if (AtomicAdd(AllocCount, 1) == 1) { + TNode* fl = Pending; + if (AtomicAdd(AllocCount, 1) == 1) { // No other allocs in progress. // If (keepCounter == PendingToFreeListCounter) then Pending was not freed by other threads. // Hence Pending is not used in any concurrent DoAlloc() atm and can be safely moved to FreeList - if (fl && keepCounter == AtomicAdd(PendingToFreeListCounter, 0) && DoCas(&Pending, (TNode*)nullptr, fl) == fl) { + if (fl && keepCounter == AtomicAdd(PendingToFreeListCounter, 0) && DoCas(&Pending, (TNode*)nullptr, fl) == fl) { // pick first element from Pending and return it - void* res = fl; + void* res = fl; fl = fl->Next; // if there are other elements in Pending list, add them to main free list FreeList(fl); AtomicAdd(PendingToFreeListCounter, 1); - AtomicAdd(AllocCount, -1); + AtomicAdd(AllocCount, -1); return res; } } - void* res = DoAlloc(); - AtomicAdd(AllocCount, -1); + void* res = DoAlloc(); + AtomicAdd(AllocCount, -1); return res; } - void* GetWholeList() { - TNode* res; + void* GetWholeList() { + TNode* res; for (res = Head; res; res = Head) { if (DoCas(&Head, (TNode*)nullptr, res) == res) break; } return res; } - void ReturnWholeList(void* ptr) { - while (AtomicAdd(AllocCount, 0) != 0) // theoretically can run into problems with parallel DoAlloc() - ; //ThreadYield(); - for (;;) { - TNode* prevHead = Head; + void ReturnWholeList(void* ptr) { + while (AtomicAdd(AllocCount, 0) != 0) // theoretically can run into problems with parallel DoAlloc() + ; //ThreadYield(); + for (;;) { + TNode* prevHead = Head; if (DoCas(&Head, (TNode*)ptr, prevHead) == prevHead) { FreeList(prevHead); break; @@ -756,14 +756,14 @@ public: ///////////////////////////////////////////////////////////////////////// static TLFAllocFreeList globalFreeLists[N_SIZES]; -static char* volatile globalCurrentPtr[N_SIZES]; +static char* volatile globalCurrentPtr[N_SIZES]; static TLFAllocFreeList blockFreeList; // globalFreeLists[] contains TFreeListGroup, each of them points up to 15 free blocks const int FL_GROUP_SIZE = 15; -struct TFreeListGroup { - TFreeListGroup* Next; - char* Ptrs[FL_GROUP_SIZE]; +struct TFreeListGroup { + TFreeListGroup* Next; + char* Ptrs[FL_GROUP_SIZE]; }; #ifdef _64_ const int FREE_LIST_GROUP_SIZEIDX = 8; @@ -774,24 +774,24 @@ const int FREE_LIST_GROUP_SIZEIDX = 6; ////////////////////////////////////////////////////////////////////////// // find free chunks and reset chunk size so they can be reused by different sized allocations // do not look at blockFreeList (TFreeListGroup has same size for any allocations) -static bool DefragmentMem() { +static bool DefragmentMem() { if (!EnableDefrag) { return false; } IncrementCounter(CT_DEGRAGMENT_CNT, 1); - int* nFreeCount = (int*)SystemAlloc(N_CHUNKS * sizeof(int)); + int* nFreeCount = (int*)SystemAlloc(N_CHUNKS * sizeof(int)); if (Y_UNLIKELY(!nFreeCount)) { //__debugbreak(); NMalloc::AbortFromCorruptedAllocator("debugbreak"); } memset(nFreeCount, 0, N_CHUNKS * sizeof(int)); - TFreeListGroup* wholeLists[N_SIZES]; + TFreeListGroup* wholeLists[N_SIZES]; for (int nSizeIdx = 0; nSizeIdx < N_SIZES; ++nSizeIdx) { wholeLists[nSizeIdx] = (TFreeListGroup*)globalFreeLists[nSizeIdx].GetWholeList(); - for (TFreeListGroup* g = wholeLists[nSizeIdx]; g; g = g->Next) { + for (TFreeListGroup* g = wholeLists[nSizeIdx]; g; g = g->Next) { for (auto pData : g->Ptrs) { if (pData) { uintptr_t nChunk = (pData - ALLOC_START) / N_CHUNK_SIZE; @@ -819,7 +819,7 @@ static bool DefragmentMem() { for (auto& wholeList : wholeLists) { TFreeListGroup** ppPtr = &wholeList; while (*ppPtr) { - TFreeListGroup* g = *ppPtr; + TFreeListGroup* g = *ppPtr; int dst = 0; for (auto pData : g->Ptrs) { if (pData) { @@ -843,7 +843,7 @@ static bool DefragmentMem() { for (uintptr_t nChunk = 0; nChunk < N_CHUNKS; ++nChunk) { if (!nFreeCount[nChunk]) continue; - char* pStart = ALLOC_START + nChunk * N_CHUNK_SIZE; + char* pStart = ALLOC_START + nChunk * N_CHUNK_SIZE; #ifdef _win_ VirtualFree(pStart, N_CHUNK_SIZE, MEM_DECOMMIT); #elif defined(_freebsd_) @@ -862,13 +862,13 @@ static bool DefragmentMem() { return bRes; } -static Y_FORCE_INLINE void* LFAllocFromCurrentChunk(int nSizeIdx, int blockSize, int count) { - char* volatile* pFreeArray = &globalCurrentPtr[nSizeIdx]; - while (char* newBlock = *pFreeArray) { - char* nextFree = newBlock + blockSize * count; +static Y_FORCE_INLINE void* LFAllocFromCurrentChunk(int nSizeIdx, int blockSize, int count) { + char* volatile* pFreeArray = &globalCurrentPtr[nSizeIdx]; + while (char* newBlock = *pFreeArray) { + char* nextFree = newBlock + blockSize * count; // check if there is space in chunk - char* globalEndPtr = ALLOC_START + ((newBlock - ALLOC_START) & ~((uintptr_t)N_CHUNK_SIZE - 1)) + N_CHUNK_SIZE; + char* globalEndPtr = ALLOC_START + ((newBlock - ALLOC_START) & ~((uintptr_t)N_CHUNK_SIZE - 1)) + N_CHUNK_SIZE; if (nextFree >= globalEndPtr) { if (nextFree > globalEndPtr) break; @@ -880,12 +880,12 @@ static Y_FORCE_INLINE void* LFAllocFromCurrentChunk(int nSizeIdx, int blockSize, return nullptr; } -enum EDefrag { +enum EDefrag { MEM_DEFRAG, NO_MEM_DEFRAG, }; -static void* SlowLFAlloc(int nSizeIdx, int blockSize, EDefrag defrag) { +static void* SlowLFAlloc(int nSizeIdx, int blockSize, EDefrag defrag) { IncrementCounter(CT_SLOW_ALLOC_CNT, 1); TLFLockHolder ls; @@ -902,9 +902,9 @@ static void* SlowLFAlloc(int nSizeIdx, int blockSize, EDefrag defrag) { for (;;) { uintptr_t nChunk; if (GetFreeChunk(&nChunk)) { - char* newPlace = ALLOC_START + nChunk * N_CHUNK_SIZE; + char* newPlace = ALLOC_START + nChunk * N_CHUNK_SIZE; #ifdef _MSC_VER - void* pTest = VirtualAlloc(newPlace, N_CHUNK_SIZE, MEM_COMMIT, PAGE_READWRITE); + void* pTest = VirtualAlloc(newPlace, N_CHUNK_SIZE, MEM_COMMIT, PAGE_READWRITE); Y_ASSERT_NOBT(pTest == newPlace); #endif chunkSizeIdx[nChunk] = (char)nSizeIdx; @@ -917,7 +917,7 @@ static void* SlowLFAlloc(int nSizeIdx, int blockSize, EDefrag defrag) { continue; } - char* largeBlock = AllocWithMMap(N_LARGE_ALLOC_SIZE, MM_NORMAL); + char* largeBlock = AllocWithMMap(N_LARGE_ALLOC_SIZE, MM_NORMAL); uintptr_t addr = ((largeBlock - ALLOC_START) + N_CHUNK_SIZE - 1) & (~(N_CHUNK_SIZE - 1)); uintptr_t endAddr = ((largeBlock - ALLOC_START) + N_LARGE_ALLOC_SIZE) & (~(N_CHUNK_SIZE - 1)); for (uintptr_t p = addr; p < endAddr; p += N_CHUNK_SIZE) { @@ -931,9 +931,9 @@ static void* SlowLFAlloc(int nSizeIdx, int blockSize, EDefrag defrag) { } // allocate single block -static Y_FORCE_INLINE void* LFAllocNoCache(int nSizeIdx, EDefrag defrag) { +static Y_FORCE_INLINE void* LFAllocNoCache(int nSizeIdx, EDefrag defrag) { int blockSize = nSizeIdxToSize[nSizeIdx]; - void* res = LFAllocFromCurrentChunk(nSizeIdx, blockSize, 1); + void* res = LFAllocFromCurrentChunk(nSizeIdx, blockSize, 1); if (res) return res; @@ -942,11 +942,11 @@ static Y_FORCE_INLINE void* LFAllocNoCache(int nSizeIdx, EDefrag defrag) { // allocate multiple blocks, returns number of blocks allocated (max FL_GROUP_SIZE) // buf should have space for at least FL_GROUP_SIZE elems -static Y_FORCE_INLINE int LFAllocNoCacheMultiple(int nSizeIdx, char** buf) { +static Y_FORCE_INLINE int LFAllocNoCacheMultiple(int nSizeIdx, char** buf) { int blockSize = nSizeIdxToSize[nSizeIdx]; - void* res = LFAllocFromCurrentChunk(nSizeIdx, blockSize, FL_GROUP_SIZE); + void* res = LFAllocFromCurrentChunk(nSizeIdx, blockSize, FL_GROUP_SIZE); if (res) { - char* resPtr = (char*)res; + char* resPtr = (char*)res; for (int k = 0; k < FL_GROUP_SIZE; ++k) { buf[k] = resPtr; resPtr += blockSize; @@ -959,9 +959,9 @@ static Y_FORCE_INLINE int LFAllocNoCacheMultiple(int nSizeIdx, char** buf) { // take several blocks from global free list (max FL_GROUP_SIZE blocks), returns number of blocks taken // buf should have space for at least FL_GROUP_SIZE elems -static Y_FORCE_INLINE int TakeBlocksFromGlobalFreeList(int nSizeIdx, char** buf) { - TLFAllocFreeList& fl = globalFreeLists[nSizeIdx]; - TFreeListGroup* g = (TFreeListGroup*)fl.Alloc(); +static Y_FORCE_INLINE int TakeBlocksFromGlobalFreeList(int nSizeIdx, char** buf) { + TLFAllocFreeList& fl = globalFreeLists[nSizeIdx]; + TFreeListGroup* g = (TFreeListGroup*)fl.Alloc(); if (g) { int resCount = 0; for (auto& ptr : g->Ptrs) { @@ -977,9 +977,9 @@ static Y_FORCE_INLINE int TakeBlocksFromGlobalFreeList(int nSizeIdx, char** buf) } // add several blocks to global free list -static Y_FORCE_INLINE void PutBlocksToGlobalFreeList(ptrdiff_t nSizeIdx, char** buf, int count) { +static Y_FORCE_INLINE void PutBlocksToGlobalFreeList(ptrdiff_t nSizeIdx, char** buf, int count) { for (int startIdx = 0; startIdx < count;) { - TFreeListGroup* g = (TFreeListGroup*)blockFreeList.Alloc(); + TFreeListGroup* g = (TFreeListGroup*)blockFreeList.Alloc(); Y_ASSERT_NOBT(sizeof(TFreeListGroup) == nSizeIdxToSize[FREE_LIST_GROUP_SIZEIDX]); if (!g) { g = (TFreeListGroup*)LFAllocNoCache(FREE_LIST_GROUP_SIZEIDX, NO_MEM_DEFRAG); @@ -994,7 +994,7 @@ static Y_FORCE_INLINE void PutBlocksToGlobalFreeList(ptrdiff_t nSizeIdx, char** g->Ptrs[i] = nullptr; // add free group to the global list - TLFAllocFreeList& fl = globalFreeLists[nSizeIdx]; + TLFAllocFreeList& fl = globalFreeLists[nSizeIdx]; fl.Free(g); startIdx += groupSize; @@ -1041,12 +1041,12 @@ struct TPerTagAllocCounter { TAtomic Size; TAtomic Count; - Y_FORCE_INLINE void Alloc(size_t size) { + Y_FORCE_INLINE void Alloc(size_t size) { AtomicAdd(Size, size); AtomicAdd(Count, 1); } - Y_FORCE_INLINE void Free(size_t size) { + Y_FORCE_INLINE void Free(size_t size) { AtomicSub(Size, size); AtomicSub(Count, 1); } @@ -1057,13 +1057,13 @@ struct TLocalPerTagAllocCounter { int Count; int Updates; - Y_FORCE_INLINE void Init() { + Y_FORCE_INLINE void Init() { Size = 0; Count = 0; Updates = 0; } - Y_FORCE_INLINE void Alloc(TPerTagAllocCounter& parent, size_t size) { + Y_FORCE_INLINE void Alloc(TPerTagAllocCounter& parent, size_t size) { Size += size; ++Count; if (++Updates > MAX_LOCAL_UPDATES) { @@ -1071,7 +1071,7 @@ struct TLocalPerTagAllocCounter { } } - Y_FORCE_INLINE void Free(TPerTagAllocCounter& parent, size_t size) { + Y_FORCE_INLINE void Free(TPerTagAllocCounter& parent, size_t size) { Size -= size; --Count; if (++Updates > MAX_LOCAL_UPDATES) { @@ -1079,11 +1079,11 @@ struct TLocalPerTagAllocCounter { } } - Y_FORCE_INLINE void Flush(TPerTagAllocCounter& parent) { - AtomicAdd(parent.Size, Size); - Size = 0; - AtomicAdd(parent.Count, Count); - Count = 0; + Y_FORCE_INLINE void Flush(TPerTagAllocCounter& parent) { + AtomicAdd(parent.Size, Size); + Size = 0; + AtomicAdd(parent.Count, Count); + Count = 0; Updates = 0; } }; @@ -1093,18 +1093,18 @@ static const int DBG_ALLOC_ALIGNED_TAG = 0xF0000000; static const int DBG_ALLOC_NUM_SIZES = 30; static TPerTagAllocCounter GlobalPerTagAllocCounters[DBG_ALLOC_MAX_TAG][DBG_ALLOC_NUM_SIZES]; -#endif // LFALLOC_DBG +#endif // LFALLOC_DBG ////////////////////////////////////////////////////////////////////////// const int THREAD_BUF = 256; static int borderSizes[N_SIZES]; const int MAX_MEM_PER_SIZE_PER_THREAD = 512 * 1024; -struct TThreadAllocInfo { +struct TThreadAllocInfo { // FreePtrs - pointers to first free blocks in per thread block list // LastFreePtrs - pointers to last blocks in lists, may be invalid if FreePtr is zero - char* FreePtrs[N_SIZES][THREAD_BUF]; + char* FreePtrs[N_SIZES][THREAD_BUF]; int FreePtrIndex[N_SIZES]; - TThreadAllocInfo* pNextInfo; + TThreadAllocInfo* pNextInfo; TLocalCounter LocalCounters[CT_MAX]; #if defined(LFALLOC_DBG) @@ -1114,7 +1114,7 @@ struct TThreadAllocInfo { HANDLE hThread; #endif - void Init(TThreadAllocInfo** pHead) { + void Init(TThreadAllocInfo** pHead) { memset(this, 0, sizeof(*this)); for (auto& i : FreePtrIndex) i = THREAD_BUF; @@ -1145,7 +1145,7 @@ struct TThreadAllocInfo { } #endif } - void Done() { + void Done() { for (auto sizeIdx : FreePtrIndex) { Y_ASSERT_NOBT(sizeIdx == THREAD_BUF); } @@ -1167,12 +1167,12 @@ struct TThreadAllocInfo { #endif } }; -PERTHREAD TThreadAllocInfo* pThreadInfo; -static TThreadAllocInfo* pThreadInfoList; +PERTHREAD TThreadAllocInfo* pThreadInfo; +static TThreadAllocInfo* pThreadInfoList; static TLFLockData LFLockThreadInfo; -static Y_FORCE_INLINE void IncrementCounter(ELFAllocCounter counter, size_t value) { +static Y_FORCE_INLINE void IncrementCounter(ELFAllocCounter counter, size_t value) { #ifdef LFALLOC_YT TThreadAllocInfo* thr = pThreadInfo; if (thr) { @@ -1183,7 +1183,7 @@ static Y_FORCE_INLINE void IncrementCounter(ELFAllocCounter counter, size_t valu #endif } -extern "C" i64 GetLFAllocCounterFast(int counter) { +extern "C" i64 GetLFAllocCounterFast(int counter) { #ifdef LFALLOC_YT return GlobalCounters[counter]; #else @@ -1191,13 +1191,13 @@ extern "C" i64 GetLFAllocCounterFast(int counter) { #endif } -extern "C" i64 GetLFAllocCounterFull(int counter) { +extern "C" i64 GetLFAllocCounterFull(int counter) { #ifdef LFALLOC_YT i64 ret = GlobalCounters[counter]; { TLFLockHolder ll(&LFLockThreadInfo); - for (TThreadAllocInfo** p = &pThreadInfoList; *p;) { - TThreadAllocInfo* pInfo = *p; + for (TThreadAllocInfo** p = &pThreadInfoList; *p;) { + TThreadAllocInfo* pInfo = *p; ret += pInfo->LocalCounters[counter].Value; p = &pInfo->pNextInfo; } @@ -1208,26 +1208,26 @@ extern "C" i64 GetLFAllocCounterFull(int counter) { #endif } -static void MoveSingleThreadFreeToGlobal(TThreadAllocInfo* pInfo) { +static void MoveSingleThreadFreeToGlobal(TThreadAllocInfo* pInfo) { for (int sizeIdx = 0; sizeIdx < N_SIZES; ++sizeIdx) { - int& freePtrIdx = pInfo->FreePtrIndex[sizeIdx]; - char** freePtrs = pInfo->FreePtrs[sizeIdx]; + int& freePtrIdx = pInfo->FreePtrIndex[sizeIdx]; + char** freePtrs = pInfo->FreePtrs[sizeIdx]; PutBlocksToGlobalFreeList(sizeIdx, freePtrs + freePtrIdx, THREAD_BUF - freePtrIdx); freePtrIdx = THREAD_BUF; } } #ifdef _win_ -static bool IsDeadThread(TThreadAllocInfo* pInfo) { +static bool IsDeadThread(TThreadAllocInfo* pInfo) { DWORD dwExit; bool isDead = !GetExitCodeThread(pInfo->hThread, &dwExit) || dwExit != STILL_ACTIVE; return isDead; } -static void CleanupAfterDeadThreads() { +static void CleanupAfterDeadThreads() { TLFLockHolder ll(&LFLockThreadInfo); - for (TThreadAllocInfo** p = &pThreadInfoList; *p;) { - TThreadAllocInfo* pInfo = *p; + for (TThreadAllocInfo** p = &pThreadInfoList; *p;) { + TThreadAllocInfo* pInfo = *p; if (IsDeadThread(pInfo)) { MoveSingleThreadFreeToGlobal(pInfo); pInfo->Done(); @@ -1241,11 +1241,11 @@ static void CleanupAfterDeadThreads() { #ifndef _win_ static pthread_key_t ThreadCacheCleaner; -static void* volatile ThreadCacheCleanerStarted; // 0 = not started, -1 = started, -2 = is starting +static void* volatile ThreadCacheCleanerStarted; // 0 = not started, -1 = started, -2 = is starting static PERTHREAD bool IsStoppingThread; -static void FreeThreadCache(void*) { - TThreadAllocInfo* pToDelete = nullptr; +static void FreeThreadCache(void*) { + TThreadAllocInfo* pToDelete = nullptr; { TLFLockHolder ll(&LFLockThreadInfo); pToDelete = pThreadInfo; @@ -1253,7 +1253,7 @@ static void FreeThreadCache(void*) { return; // remove from the list - for (TThreadAllocInfo** p = &pThreadInfoList; *p; p = &(*p)->pNextInfo) { + for (TThreadAllocInfo** p = &pThreadInfoList; *p; p = &(*p)->pNextInfo) { if (*p == pToDelete) { *p = pToDelete->pNextInfo; break; @@ -1270,7 +1270,7 @@ static void FreeThreadCache(void*) { } #endif -static void AllocThreadInfo() { +static void AllocThreadInfo() { #ifndef _win_ if (DoCas(&ThreadCacheCleanerStarted, (void*)-2, (void*)nullptr) == (void*)nullptr) { pthread_key_create(&ThreadCacheCleaner, FreeThreadCache); @@ -1300,9 +1300,9 @@ static void AllocThreadInfo() { #endif } - ////////////////////////////////////////////////////////////////////////// - // DBG stuff - ////////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + // DBG stuff + ////////////////////////////////////////////////////////////////////////// #if defined(LFALLOC_DBG) @@ -1315,11 +1315,11 @@ struct TAllocHeader { // should be power of 2 static_assert(sizeof(TAllocHeader) == 16); -static inline void* GetAllocPtr(TAllocHeader* p) { +static inline void* GetAllocPtr(TAllocHeader* p) { return p + 1; } -static inline TAllocHeader* GetAllocHeader(void* p) { +static inline TAllocHeader* GetAllocHeader(void* p) { auto* header = ((TAllocHeader*)p) - 1; if (header->Tag == DBG_ALLOC_ALIGNED_TAG) { return (TAllocHeader*)header->Size; @@ -1329,7 +1329,7 @@ static inline TAllocHeader* GetAllocHeader(void* p) { } PERTHREAD int AllocationTag; -extern "C" int SetThreadAllocTag(int tag) { +extern "C" int SetThreadAllocTag(int tag) { int prevTag = AllocationTag; if (tag < DBG_ALLOC_MAX_TAG && tag >= 0) { AllocationTag = tag; @@ -1338,7 +1338,7 @@ extern "C" int SetThreadAllocTag(int tag) { } PERTHREAD bool ProfileCurrentThread; -extern "C" bool SetProfileCurrentThread(bool newVal) { +extern "C" bool SetProfileCurrentThread(bool newVal) { bool prevVal = ProfileCurrentThread; ProfileCurrentThread = newVal; return prevVal; @@ -1352,21 +1352,21 @@ extern "C" bool SetProfileAllThreads(bool newVal) { } static volatile bool AllocationSamplingEnabled; -extern "C" bool SetAllocationSamplingEnabled(bool newVal) { +extern "C" bool SetAllocationSamplingEnabled(bool newVal) { bool prevVal = AllocationSamplingEnabled; AllocationSamplingEnabled = newVal; return prevVal; } static size_t AllocationSampleRate = 1000; -extern "C" size_t SetAllocationSampleRate(size_t newVal) { +extern "C" size_t SetAllocationSampleRate(size_t newVal) { size_t prevVal = AllocationSampleRate; AllocationSampleRate = newVal; return prevVal; } static size_t AllocationSampleMaxSize = N_MAX_FAST_SIZE; -extern "C" size_t SetAllocationSampleMaxSize(size_t newVal) { +extern "C" size_t SetAllocationSampleMaxSize(size_t newVal) { size_t prevVal = AllocationSampleMaxSize; AllocationSampleMaxSize = newVal; return prevVal; @@ -1374,7 +1374,7 @@ extern "C" size_t SetAllocationSampleMaxSize(size_t newVal) { using TAllocationCallback = int(int tag, size_t size, int sizeIdx); static TAllocationCallback* AllocationCallback; -extern "C" TAllocationCallback* SetAllocationCallback(TAllocationCallback* newVal) { +extern "C" TAllocationCallback* SetAllocationCallback(TAllocationCallback* newVal) { TAllocationCallback* prevVal = AllocationCallback; AllocationCallback = newVal; return prevVal; @@ -1382,7 +1382,7 @@ extern "C" TAllocationCallback* SetAllocationCallback(TAllocationCallback* newVa using TDeallocationCallback = void(int cookie, int tag, size_t size, int sizeIdx); static TDeallocationCallback* DeallocationCallback; -extern "C" TDeallocationCallback* SetDeallocationCallback(TDeallocationCallback* newVal) { +extern "C" TDeallocationCallback* SetDeallocationCallback(TDeallocationCallback* newVal) { TDeallocationCallback* prevVal = DeallocationCallback; DeallocationCallback = newVal; return prevVal; @@ -1392,7 +1392,7 @@ PERTHREAD TAtomic AllocationsCount; PERTHREAD bool InAllocationCallback; static const int DBG_ALLOC_INVALID_COOKIE = -1; -static inline int SampleAllocation(TAllocHeader* p, int sizeIdx) { +static inline int SampleAllocation(TAllocHeader* p, int sizeIdx) { int cookie = DBG_ALLOC_INVALID_COOKIE; if (AllocationSamplingEnabled && (ProfileCurrentThread || ProfileAllThreads) && !InAllocationCallback) { if (p->Size > AllocationSampleMaxSize || ++AllocationsCount % AllocationSampleRate == 0) { @@ -1406,7 +1406,7 @@ static inline int SampleAllocation(TAllocHeader* p, int sizeIdx) { return cookie; } -static inline void SampleDeallocation(TAllocHeader* p, int sizeIdx) { +static inline void SampleDeallocation(TAllocHeader* p, int sizeIdx) { if (p->Cookie != DBG_ALLOC_INVALID_COOKIE && !InAllocationCallback) { if (DeallocationCallback) { InAllocationCallback = true; @@ -1416,7 +1416,7 @@ static inline void SampleDeallocation(TAllocHeader* p, int sizeIdx) { } } -static inline void TrackPerTagAllocation(TAllocHeader* p, int sizeIdx) { +static inline void TrackPerTagAllocation(TAllocHeader* p, int sizeIdx) { if (p->Tag < DBG_ALLOC_MAX_TAG && p->Tag >= 0) { Y_ASSERT_NOBT(sizeIdx < DBG_ALLOC_NUM_SIZES); auto& global = GlobalPerTagAllocCounters[p->Tag][sizeIdx]; @@ -1431,7 +1431,7 @@ static inline void TrackPerTagAllocation(TAllocHeader* p, int sizeIdx) { } } -static inline void TrackPerTagDeallocation(TAllocHeader* p, int sizeIdx) { +static inline void TrackPerTagDeallocation(TAllocHeader* p, int sizeIdx) { if (p->Tag < DBG_ALLOC_MAX_TAG && p->Tag >= 0) { Y_ASSERT_NOBT(sizeIdx < DBG_ALLOC_NUM_SIZES); auto& global = GlobalPerTagAllocCounters[p->Tag][sizeIdx]; @@ -1446,7 +1446,7 @@ static inline void TrackPerTagDeallocation(TAllocHeader* p, int sizeIdx) { } } -static void* TrackAllocation(void* ptr, size_t size, int sizeIdx) { +static void* TrackAllocation(void* ptr, size_t size, int sizeIdx) { TAllocHeader* p = (TAllocHeader*)ptr; p->Size = size; p->Tag = AllocationTag; @@ -1455,7 +1455,7 @@ static void* TrackAllocation(void* ptr, size_t size, int sizeIdx) { return GetAllocPtr(p); } -static void TrackDeallocation(void* ptr, int sizeIdx) { +static void TrackDeallocation(void* ptr, int sizeIdx) { TAllocHeader* p = (TAllocHeader*)ptr; SampleDeallocation(p, sizeIdx); TrackPerTagDeallocation(p, sizeIdx); @@ -1470,15 +1470,15 @@ extern "C" void GetPerTagAllocInfo( bool flushPerThreadCounters, TPerTagAllocInfo* info, int& maxTag, - int& numSizes) { + int& numSizes) { maxTag = DBG_ALLOC_MAX_TAG; numSizes = DBG_ALLOC_NUM_SIZES; if (info) { if (flushPerThreadCounters) { TLFLockHolder ll(&LFLockThreadInfo); - for (TThreadAllocInfo** p = &pThreadInfoList; *p;) { - TThreadAllocInfo* pInfo = *p; + for (TThreadAllocInfo** p = &pThreadInfoList; *p;) { + TThreadAllocInfo* pInfo = *p; for (int tag = 0; tag < DBG_ALLOC_MAX_TAG; ++tag) { for (int sizeIdx = 0; sizeIdx < DBG_ALLOC_NUM_SIZES; ++sizeIdx) { auto& local = pInfo->LocalPerTagAllocCounters[tag][sizeIdx]; @@ -1493,7 +1493,7 @@ extern "C" void GetPerTagAllocInfo( for (int tag = 0; tag < DBG_ALLOC_MAX_TAG; ++tag) { for (int sizeIdx = 0; sizeIdx < DBG_ALLOC_NUM_SIZES; ++sizeIdx) { auto& global = GlobalPerTagAllocCounters[tag][sizeIdx]; - auto& res = info[tag * DBG_ALLOC_NUM_SIZES + sizeIdx]; + auto& res = info[tag * DBG_ALLOC_NUM_SIZES + sizeIdx]; res.Count = global.Count; res.Size = global.Size; } @@ -1501,7 +1501,7 @@ extern "C" void GetPerTagAllocInfo( } } -#endif // LFALLOC_DBG +#endif // LFALLOC_DBG ////////////////////////////////////////////////////////////////////////// static Y_FORCE_INLINE void* LFAllocImpl(size_t _nSize) { @@ -1528,7 +1528,7 @@ static Y_FORCE_INLINE void* LFAllocImpl(size_t _nSize) { IncrementCounter(CT_SMALL_ALLOC, nSizeIdxToSize[nSizeIdx]); // check per thread buffer - TThreadAllocInfo* thr = pThreadInfo; + TThreadAllocInfo* thr = pThreadInfo; if (!thr) { AllocThreadInfo(); thr = pThreadInfo; @@ -1541,7 +1541,7 @@ static Y_FORCE_INLINE void* LFAllocImpl(size_t _nSize) { } } { - int& freePtrIdx = thr->FreePtrIndex[nSizeIdx]; + int& freePtrIdx = thr->FreePtrIndex[nSizeIdx]; if (freePtrIdx < THREAD_BUF) { void* ptr = thr->FreePtrs[nSizeIdx][freePtrIdx++]; #if defined(LFALLOC_DBG) @@ -1551,7 +1551,7 @@ static Y_FORCE_INLINE void* LFAllocImpl(size_t _nSize) { } // try to alloc from global free list - char* buf[FL_GROUP_SIZE]; + char* buf[FL_GROUP_SIZE]; int count = TakeBlocksFromGlobalFreeList(nSizeIdx, buf); if (count == 0) { count = LFAllocNoCacheMultiple(nSizeIdx, buf); @@ -1559,7 +1559,7 @@ static Y_FORCE_INLINE void* LFAllocImpl(size_t _nSize) { NMalloc::AbortFromCorruptedAllocator("no way LFAllocNoCacheMultiple() can fail"); } } - char** dstBuf = thr->FreePtrs[nSizeIdx] + freePtrIdx - 1; + char** dstBuf = thr->FreePtrs[nSizeIdx] + freePtrIdx - 1; for (int i = 0; i < count - 1; ++i) dstBuf[-i] = buf[i]; freePtrIdx -= count - 1; @@ -1581,7 +1581,7 @@ static Y_FORCE_INLINE void* LFAlloc(size_t _nSize) { return res; } -static Y_FORCE_INLINE void LFFree(void* p) { +static Y_FORCE_INLINE void LFFree(void* p) { #if defined(LFALLOC_DBG) if (p == nullptr) return; @@ -1620,9 +1620,9 @@ static Y_FORCE_INLINE void LFFree(void* p) { IncrementCounter(CT_SMALL_FREE, nSizeIdxToSize[nSizeIdx]); // try to store info to per thread buf - TThreadAllocInfo* thr = pThreadInfo; + TThreadAllocInfo* thr = pThreadInfo; if (thr) { - int& freePtrIdx = thr->FreePtrIndex[nSizeIdx]; + int& freePtrIdx = thr->FreePtrIndex[nSizeIdx]; if (freePtrIdx > borderSizes[nSizeIdx]) { thr->FreePtrs[nSizeIdx][--freePtrIdx] = (char*)p; return; @@ -1632,7 +1632,7 @@ static Y_FORCE_INLINE void LFFree(void* p) { int freeCount = FL_GROUP_SIZE; if (freeCount > THREAD_BUF - freePtrIdx) freeCount = THREAD_BUF - freePtrIdx; - char** freePtrs = thr->FreePtrs[nSizeIdx]; + char** freePtrs = thr->FreePtrs[nSizeIdx]; PutBlocksToGlobalFreeList(nSizeIdx, freePtrs + freePtrIdx, freeCount); freePtrIdx += freeCount; @@ -1667,7 +1667,7 @@ static size_t LFGetSize(const void* p) { //////////////////////////////////////////////////////////////////////////////////////////////////// // Output mem alloc stats const int N_PAGE_SIZE = 4096; -static void DebugTraceMMgr(const char* pszFormat, ...) // __cdecl +static void DebugTraceMMgr(const char* pszFormat, ...) // __cdecl { static char buff[20000]; va_list va; @@ -1675,21 +1675,21 @@ static void DebugTraceMMgr(const char* pszFormat, ...) // __cdecl va_start(va, pszFormat); vsprintf(buff, pszFormat, va); va_end(va); -// -#ifdef _win_ +// +#ifdef _win_ OutputDebugStringA(buff); -#else +#else fputs(buff, stderr); -#endif +#endif } -struct TChunkStats { +struct TChunkStats { char *Start, *Finish; - i64 Size; - char* Entries; - i64 FreeCount; + i64 Size; + char* Entries; + i64 FreeCount; - TChunkStats(size_t chunk, i64 size, char* entries) + TChunkStats(size_t chunk, i64 size, char* entries) : Size(size) , Entries(entries) , FreeCount(0) @@ -1697,17 +1697,17 @@ struct TChunkStats { Start = ALLOC_START + chunk * N_CHUNK_SIZE; Finish = Start + N_CHUNK_SIZE; } - void CheckBlock(char* pBlock) { + void CheckBlock(char* pBlock) { if (pBlock && pBlock >= Start && pBlock < Finish) { ++FreeCount; - i64 nShift = pBlock - Start; - i64 nOffsetInStep = nShift & (N_CHUNK_SIZE - 1); + i64 nShift = pBlock - Start; + i64 nOffsetInStep = nShift & (N_CHUNK_SIZE - 1); Entries[nOffsetInStep / Size] = 1; } } - void SetGlobalFree(char* ptr) { - i64 nShift = ptr - Start; - i64 nOffsetInStep = nShift & (N_CHUNK_SIZE - 1); + void SetGlobalFree(char* ptr) { + i64 nShift = ptr - Start; + i64 nOffsetInStep = nShift & (N_CHUNK_SIZE - 1); while (nOffsetInStep + Size <= N_CHUNK_SIZE) { ++FreeCount; Entries[nOffsetInStep / Size] = 1; @@ -1716,17 +1716,17 @@ struct TChunkStats { } }; -static void DumpMemoryBlockUtilizationLocked() { - TFreeListGroup* wholeLists[N_SIZES]; +static void DumpMemoryBlockUtilizationLocked() { + TFreeListGroup* wholeLists[N_SIZES]; for (int nSizeIdx = 0; nSizeIdx < N_SIZES; ++nSizeIdx) { wholeLists[nSizeIdx] = (TFreeListGroup*)globalFreeLists[nSizeIdx].GetWholeList(); } - char* bfList = (char*)blockFreeList.GetWholeList(); + char* bfList = (char*)blockFreeList.GetWholeList(); DebugTraceMMgr("memory blocks utilisation stats:\n"); - i64 nTotalAllocated = 0, nTotalFree = 0, nTotalBadPages = 0, nTotalPages = 0, nTotalUsed = 0, nTotalLocked = 0; - i64 nTotalGroupBlocks = 0; - char* entries; + i64 nTotalAllocated = 0, nTotalFree = 0, nTotalBadPages = 0, nTotalPages = 0, nTotalUsed = 0, nTotalLocked = 0; + i64 nTotalGroupBlocks = 0; + char* entries; entries = (char*)SystemAlloc((N_CHUNK_SIZE / 4)); for (size_t k = 0; k < N_CHUNKS; ++k) { if (chunkSizeIdx[k] <= 0) { @@ -1734,12 +1734,12 @@ static void DumpMemoryBlockUtilizationLocked() { nTotalLocked += N_CHUNK_SIZE; continue; } - i64 nSizeIdx = chunkSizeIdx[k]; - i64 nSize = nSizeIdxToSize[nSizeIdx]; + i64 nSizeIdx = chunkSizeIdx[k]; + i64 nSize = nSizeIdxToSize[nSizeIdx]; TChunkStats cs(k, nSize, entries); int nEntriesTotal = N_CHUNK_SIZE / nSize; memset(entries, 0, nEntriesTotal); - for (TFreeListGroup* g = wholeLists[nSizeIdx]; g; g = g->Next) { + for (TFreeListGroup* g = wholeLists[nSizeIdx]; g; g = g->Next) { for (auto& ptr : g->Ptrs) cs.CheckBlock(ptr); } @@ -1749,7 +1749,7 @@ static void DumpMemoryBlockUtilizationLocked() { for (; g; g = g->Next) csGB.CheckBlock((char*)g); } - for (char* blk = bfList; blk; blk = *(char**)blk) + for (char* blk = bfList; blk; blk = *(char**)blk) csGB.CheckBlock(blk); nTotalGroupBlocks += csGB.FreeCount * nSize; } @@ -1765,25 +1765,25 @@ static void DumpMemoryBlockUtilizationLocked() { nBit = 1; // free entry else nBit = 2; // used entry - for (i64 nDelta = nSize - 1; nDelta >= 0; nDelta -= N_PAGE_SIZE) + for (i64 nDelta = nSize - 1; nDelta >= 0; nDelta -= N_PAGE_SIZE) pages[(nShift + nDelta) / N_PAGE_SIZE] |= nBit; } - i64 nBadPages = 0; + i64 nBadPages = 0; for (auto page : pages) { nBadPages += page == 3; nTotalPages += page != 1; } DebugTraceMMgr("entry = %lld; size = %lld; free = %lld; system %lld; utilisation = %g%%, fragmentation = %g%%\n", - k, nSize, cs.FreeCount * nSize, csGB.FreeCount * nSize, - (N_CHUNK_SIZE - cs.FreeCount * nSize) * 100.0f / N_CHUNK_SIZE, 100.0f * nBadPages / Y_ARRAY_SIZE(pages)); + k, nSize, cs.FreeCount * nSize, csGB.FreeCount * nSize, + (N_CHUNK_SIZE - cs.FreeCount * nSize) * 100.0f / N_CHUNK_SIZE, 100.0f * nBadPages / Y_ARRAY_SIZE(pages)); nTotalAllocated += N_CHUNK_SIZE; nTotalFree += cs.FreeCount * nSize; nTotalBadPages += nBadPages; } SystemFree(entries); DebugTraceMMgr("Total allocated = %llu, free = %lld, system = %lld, locked for future use %lld, utilisation = %g, fragmentation = %g\n", - nTotalAllocated, nTotalFree, nTotalGroupBlocks, nTotalLocked, - 100.0f * (nTotalAllocated - nTotalFree) / nTotalAllocated, 100.0f * nTotalBadPages / nTotalPages); + nTotalAllocated, nTotalFree, nTotalGroupBlocks, nTotalLocked, + 100.0f * (nTotalAllocated - nTotalFree) / nTotalAllocated, 100.0f * nTotalBadPages / nTotalPages); DebugTraceMMgr("Total %lld bytes used, %lld bytes in used pages\n", nTotalUsed, nTotalPages * N_PAGE_SIZE); for (int nSizeIdx = 0; nSizeIdx < N_SIZES; ++nSizeIdx) @@ -1791,12 +1791,12 @@ static void DumpMemoryBlockUtilizationLocked() { blockFreeList.ReturnWholeList(bfList); } -void FlushThreadFreeList() { +void FlushThreadFreeList() { if (pThreadInfo) MoveSingleThreadFreeToGlobal(pThreadInfo); } -void DumpMemoryBlockUtilization() { +void DumpMemoryBlockUtilization() { // move current thread free to global lists to get better statistics FlushThreadFreeList(); { @@ -1845,19 +1845,19 @@ static const char* LFAlloc_GetParam(const char* param) { }; static const TParam Params[] = { - {"GetLFAllocCounterFast", (const char*)&GetLFAllocCounterFast}, - {"GetLFAllocCounterFull", (const char*)&GetLFAllocCounterFull}, + {"GetLFAllocCounterFast", (const char*)&GetLFAllocCounterFast}, + {"GetLFAllocCounterFull", (const char*)&GetLFAllocCounterFull}, #if defined(LFALLOC_DBG) - {"SetThreadAllocTag", (const char*)&SetThreadAllocTag}, - {"SetProfileCurrentThread", (const char*)&SetProfileCurrentThread}, + {"SetThreadAllocTag", (const char*)&SetThreadAllocTag}, + {"SetProfileCurrentThread", (const char*)&SetProfileCurrentThread}, {"SetProfileAllThreads", (const char*)&SetProfileAllThreads}, - {"SetAllocationSamplingEnabled", (const char*)&SetAllocationSamplingEnabled}, - {"SetAllocationSampleRate", (const char*)&SetAllocationSampleRate}, - {"SetAllocationSampleMaxSize", (const char*)&SetAllocationSampleMaxSize}, - {"SetAllocationCallback", (const char*)&SetAllocationCallback}, - {"SetDeallocationCallback", (const char*)&SetDeallocationCallback}, - {"GetPerTagAllocInfo", (const char*)&GetPerTagAllocInfo}, -#endif // LFALLOC_DBG + {"SetAllocationSamplingEnabled", (const char*)&SetAllocationSamplingEnabled}, + {"SetAllocationSampleRate", (const char*)&SetAllocationSampleRate}, + {"SetAllocationSampleMaxSize", (const char*)&SetAllocationSampleMaxSize}, + {"SetAllocationCallback", (const char*)&SetAllocationCallback}, + {"SetDeallocationCallback", (const char*)&SetDeallocationCallback}, + {"GetPerTagAllocInfo", (const char*)&GetPerTagAllocInfo}, +#endif // LFALLOC_DBG }; for (int i = 0; i < Y_ARRAY_SIZE(Params); ++i) { diff --git a/library/cpp/lfalloc/ya.make b/library/cpp/lfalloc/ya.make index cace05f9d8..efe2fa7eeb 100644 --- a/library/cpp/lfalloc/ya.make +++ b/library/cpp/lfalloc/ya.make @@ -1,21 +1,21 @@ -LIBRARY() - +LIBRARY() + OWNER(gulin) NO_UTIL() NO_COMPILER_WARNINGS() -IF (ARCH_AARCH64) - PEERDIR( - contrib/libs/jemalloc - ) -ELSE() - SRCS( - lf_allocX64.cpp - ) +IF (ARCH_AARCH64) + PEERDIR( + contrib/libs/jemalloc + ) +ELSE() + SRCS( + lf_allocX64.cpp + ) ENDIF() - + PEERDIR( library/cpp/malloc/api ) diff --git a/library/cpp/lfalloc/yt/ya.make b/library/cpp/lfalloc/yt/ya.make index 8c1a4f8a72..122cae9564 100644 --- a/library/cpp/lfalloc/yt/ya.make +++ b/library/cpp/lfalloc/yt/ya.make @@ -6,20 +6,20 @@ NO_UTIL() NO_COMPILER_WARNINGS() -IF (ARCH_AARCH64) - PEERDIR( - contrib/libs/jemalloc - ) -ELSE() - IF ("${YMAKE}" MATCHES "devtools") - CFLAGS(-DYMAKE=1) - ENDIF() +IF (ARCH_AARCH64) + PEERDIR( + contrib/libs/jemalloc + ) +ELSE() + IF ("${YMAKE}" MATCHES "devtools") + CFLAGS(-DYMAKE=1) + ENDIF() CXXFLAGS(-DLFALLOC_YT) - SRCS( + SRCS( ../lf_allocX64.cpp - ) + ) ENDIF() - + PEERDIR( library/cpp/malloc/api ) diff --git a/library/cpp/linear_regression/benchmark/pool.cpp b/library/cpp/linear_regression/benchmark/pool.cpp index 7f2c6a7004..12b547be89 100644 --- a/library/cpp/linear_regression/benchmark/pool.cpp +++ b/library/cpp/linear_regression/benchmark/pool.cpp @@ -51,7 +51,7 @@ bool TPool::TCVIterator::IsValid() const { return Current != InstanceFoldNumbers.end(); } -const TInstance& TPool::TCVIterator::operator*() const { +const TInstance& TPool::TCVIterator::operator*() const { return ParentPool[Current - InstanceFoldNumbers.begin()]; } @@ -75,10 +75,10 @@ void TPool::TCVIterator::Advance() { bool TPool::TCVIterator::TakeCurrent() const { switch (IteratorType) { - case LearnIterator: - return *Current != TestFoldNumber; - case TestIterator: - return *Current == TestFoldNumber; + case LearnIterator: + return *Current != TestFoldNumber; + case TestIterator: + return *Current == TestFoldNumber; } return false; } diff --git a/library/cpp/linear_regression/benchmark/pool.h b/library/cpp/linear_regression/benchmark/pool.h index 43288319c8..4dcf7d7e9e 100644 --- a/library/cpp/linear_regression/benchmark/pool.h +++ b/library/cpp/linear_regression/benchmark/pool.h @@ -14,7 +14,7 @@ struct TInstance { static TInstance FromFeaturesString(const TString& featuresString); }; -struct TPool: public TVector<TInstance> { +struct TPool: public TVector<TInstance> { enum EIteratorType { LearnIterator, TestIterator, @@ -33,7 +33,7 @@ struct TPool: public TVector<TInstance> { const size_t* Current; TMersenne<ui64> RandomGenerator; - + public: TCVIterator(const TPool& parentPool, const size_t foldsCount, @@ -45,10 +45,10 @@ struct TPool: public TVector<TInstance> { bool IsValid() const; - const TInstance& operator*() const; - const TInstance* operator->() const; + const TInstance& operator*() const; + const TInstance* operator->() const; TPool::TCVIterator& operator++(); - + private: void Advance(); bool TakeCurrent() const; diff --git a/library/cpp/linear_regression/linear_model.h b/library/cpp/linear_regression/linear_model.h index 8bb050cff7..551bfe36dc 100644 --- a/library/cpp/linear_regression/linear_model.h +++ b/library/cpp/linear_regression/linear_model.h @@ -8,35 +8,35 @@ #include <utility> class TLinearModel { -private: - TVector<double> Coefficients; - double Intercept; - -public: - Y_SAVELOAD_DEFINE(Coefficients, Intercept); - - TLinearModel(TVector<double>&& coefficients, const double intercept) - : Coefficients(std::move(coefficients)) - , Intercept(intercept) - { - } - - explicit TLinearModel(size_t featuresCount = 0) - : Coefficients(featuresCount) - , Intercept(0.) - { - } - - const TVector<double>& GetCoefficients() const { - return Coefficients; - } - - double GetIntercept() const { - return Intercept; - } - - template <typename T> - double Prediction(const TVector<T>& features) const { - return InnerProduct(Coefficients, features, Intercept); - } +private: + TVector<double> Coefficients; + double Intercept; + +public: + Y_SAVELOAD_DEFINE(Coefficients, Intercept); + + TLinearModel(TVector<double>&& coefficients, const double intercept) + : Coefficients(std::move(coefficients)) + , Intercept(intercept) + { + } + + explicit TLinearModel(size_t featuresCount = 0) + : Coefficients(featuresCount) + , Intercept(0.) + { + } + + const TVector<double>& GetCoefficients() const { + return Coefficients; + } + + double GetIntercept() const { + return Intercept; + } + + template <typename T> + double Prediction(const TVector<T>& features) const { + return InnerProduct(Coefficients, features, Intercept); + } }; diff --git a/library/cpp/linear_regression/linear_regression.cpp b/library/cpp/linear_regression/linear_regression.cpp index 150f9d214e..18ae939893 100644 --- a/library/cpp/linear_regression/linear_regression.cpp +++ b/library/cpp/linear_regression/linear_regression.cpp @@ -4,8 +4,8 @@ #include <util/generic/ymath.h> #ifdef _sse2_ -#include <emmintrin.h> -#include <xmmintrin.h> +#include <emmintrin.h> +#include <xmmintrin.h> #endif #include <algorithm> @@ -68,8 +68,8 @@ bool TLinearRegressionSolver::Add(const TVector<double>& features, const double LastMeans[featureNumber] = weight * (feature - featureMean); featureMean += weight * (feature - featureMean) / SumWeights.Get(); - NewMeans[featureNumber] = feature - featureMean; - ; + NewMeans[featureNumber] = feature - featureMean; + ; } double* olsMatrixElement = LinearizedOLSMatrix.data(); @@ -169,7 +169,7 @@ bool TSLRSolver::Add(const double feature, const double goal, const double weigh bool TSLRSolver::Add(const double* featuresBegin, const double* featuresEnd, - const double* goalsBegin) { + const double* goalsBegin) { for (; featuresBegin != featuresEnd; ++featuresBegin, ++goalsBegin) { Add(*featuresBegin, *goalsBegin); } @@ -178,7 +178,7 @@ bool TSLRSolver::Add(const double* featuresBegin, bool TSLRSolver::Add(const double* featuresBegin, const double* featuresEnd, const double* goalsBegin, - const double* weightsBegin) { + const double* weightsBegin) { for (; featuresBegin != featuresEnd; ++featuresBegin, ++goalsBegin, ++weightsBegin) { Add(*featuresBegin, *goalsBegin, *weightsBegin); } @@ -198,7 +198,7 @@ namespace { const double regularizationThreshold, const double regularizationParameter, TVector<double>& decompositionTrace, - TVector<TVector<double>>& decompositionMatrix) { + TVector<TVector<double>>& decompositionMatrix) { const size_t featuresCount = decompositionTrace.size(); size_t olsMatrixElementIdx = 0; @@ -239,7 +239,7 @@ namespace { void LDLDecomposition(const TVector<double>& linearizedOLSMatrix, TVector<double>& decompositionTrace, - TVector<TVector<double>>& decompositionMatrix) { + TVector<TVector<double>>& decompositionMatrix) { const double regularizationThreshold = 1e-5; double regularizationParameter = 0.; @@ -247,14 +247,14 @@ namespace { regularizationThreshold, regularizationParameter, decompositionTrace, - decompositionMatrix)) { + decompositionMatrix)) { regularizationParameter = regularizationParameter ? 2 * regularizationParameter : 1e-5; } } - TVector<double> SolveLower(const TVector<TVector<double>>& decompositionMatrix, + TVector<double> SolveLower(const TVector<TVector<double>>& decompositionMatrix, const TVector<double>& decompositionTrace, - const TVector<double>& olsVector) { + const TVector<double>& olsVector) { const size_t featuresCount = olsVector.size(); TVector<double> solution(featuresCount); @@ -275,8 +275,8 @@ namespace { return solution; } - TVector<double> SolveUpper(const TVector<TVector<double>>& decompositionMatrix, - const TVector<double>& lowerSolution) { + TVector<double> SolveUpper(const TVector<TVector<double>>& decompositionMatrix, + const TVector<double>& lowerSolution) { const size_t featuresCount = lowerSolution.size(); TVector<double> solution(featuresCount); @@ -297,7 +297,7 @@ namespace { const size_t featuresCount = olsVector.size(); TVector<double> decompositionTrace(featuresCount); - TVector<TVector<double>> decompositionMatrix(featuresCount, TVector<double>(featuresCount)); + TVector<TVector<double>> decompositionMatrix(featuresCount, TVector<double>(featuresCount)); LDLDecomposition(olsMatrix, decompositionTrace, decompositionMatrix); @@ -307,7 +307,7 @@ namespace { double SumSquaredErrors(const TVector<double>& olsMatrix, const TVector<double>& olsVector, const TVector<double>& solution, - const double goalsDeviation) { + const double goalsDeviation) { const size_t featuresCount = olsVector.size(); double sumSquaredErrors = goalsDeviation; @@ -335,7 +335,7 @@ namespace { for (; leftFeature != featuresEnd; ++leftFeature, ++matrixElement) { const double weightedFeature = weight * *leftFeature; const double* rightFeature = leftFeature; - __m128d wf = {weightedFeature, weightedFeature}; + __m128d wf = {weightedFeature, weightedFeature}; for (size_t i = 0; i < unaligned; ++i, ++rightFeature, ++matrixElement) { *matrixElement += weightedFeature * *rightFeature; } @@ -397,10 +397,10 @@ namespace { TFeaturesTransformer TFeaturesTransformerLearner::Solve(const size_t iterationsCount /* = 100 */) { TTransformationParameters transformationParameters; - auto updateParameter = [this, &transformationParameters](double TTransformationParameters::*parameter, + auto updateParameter = [this, &transformationParameters](double TTransformationParameters::*parameter, const double left, - const double right) { - auto evalParameter = [this, &transformationParameters, parameter](double parameterValue) { + const double right) { + auto evalParameter = [this, &transformationParameters, parameter](double parameterValue) { transformationParameters.*parameter = parameterValue; TFeaturesTransformer transformer(TransformationType, transformationParameters); diff --git a/library/cpp/linear_regression/linear_regression.h b/library/cpp/linear_regression/linear_regression.h index e57de5ff6c..4a2684f506 100644 --- a/library/cpp/linear_regression/linear_regression.h +++ b/library/cpp/linear_regression/linear_regression.h @@ -15,7 +15,7 @@ private: TVector<double> LinearizedOLSMatrix; TVector<double> OLSVector; - + public: bool Add(const TVector<double>& features, const double goal, const double weight = 1.); TLinearModel Solve() const; @@ -35,7 +35,7 @@ private: TVector<double> OLSVector; TKahanAccumulator<double> SumWeights; - + public: bool Add(const TVector<double>& features, const double goal, const double weight = 1.); TLinearModel Solve() const; @@ -54,7 +54,7 @@ private: TStoreType SumProducts = TStoreType(); TStoreType SumWeights = TStoreType(); - + public: bool Add(const double feature, const double goal, const double weight = 1.) { SumFeatures += feature * weight; @@ -72,7 +72,7 @@ public: template <typename TFloatType> void Solve(TFloatType& factor, TFloatType& intercept, const double regularizationParameter = 0.1) const { - if (!(double)SumGoals) { + if (!(double)SumGoals) { factor = intercept = TFloatType(); return; } @@ -82,20 +82,20 @@ public: if (!featuresDeviation) { factor = TFloatType(); - intercept = (double)SumGoals / (double)SumWeights; + intercept = (double)SumGoals / (double)SumWeights; return; } factor = productsDeviation / (featuresDeviation + regularizationParameter); - intercept = (double)SumGoals / (double)SumWeights - factor * (double)SumFeatures / (double)SumWeights; + intercept = (double)SumGoals / (double)SumWeights - factor * (double)SumFeatures / (double)SumWeights; } double SumSquaredErrors(const double regularizationParameter = 0.1) const { - if (!(double)SumWeights) { + if (!(double)SumWeights) { return 0.; } - const double sumGoalSquaredDeviations = (double)SumSquaredGoals - (double)SumGoals / (double)SumWeights * (double)SumGoals; + const double sumGoalSquaredDeviations = (double)SumSquaredGoals - (double)SumGoals / (double)SumWeights * (double)SumGoals; double productsDeviation, featuresDeviation; SetupSolutionFactors(productsDeviation, featuresDeviation); @@ -108,24 +108,24 @@ public: const double sumSquaredErrors = factor * factor * featuresDeviation - 2 * factor * productsDeviation + sumGoalSquaredDeviations; return Max(0., sumSquaredErrors); } - + private: void SetupSolutionFactors(double& productsDeviation, double& featuresDeviation) const { - if (!(double)SumWeights) { + if (!(double)SumWeights) { productsDeviation = featuresDeviation = 0.; return; } - featuresDeviation = (double)SumSquaredFeatures - (double)SumFeatures / (double)SumWeights * (double)SumFeatures; + featuresDeviation = (double)SumSquaredFeatures - (double)SumFeatures / (double)SumWeights * (double)SumFeatures; if (!featuresDeviation) { return; } - productsDeviation = (double)SumProducts - (double)SumFeatures / (double)SumWeights * (double)SumGoals; + productsDeviation = (double)SumProducts - (double)SumFeatures / (double)SumWeights * (double)SumGoals; } }; using TFastSLRSolver = TTypedFastSLRSolver<double>; -using TKahanSLRSolver = TTypedFastSLRSolver<TKahanAccumulator<double>>; +using TKahanSLRSolver = TTypedFastSLRSolver<TKahanAccumulator<double>>; class TSLRSolver { private: @@ -138,7 +138,7 @@ private: TKahanAccumulator<double> SumWeights; double Covariation = 0.; - + public: bool Add(const double feature, const double goal, const double weight = 1.); @@ -178,7 +178,7 @@ template <typename TSLRSolverType> class TTypedBestSLRSolver { private: TVector<TSLRSolverType> SLRSolvers; - + public: bool Add(const TVector<double>& features, const double goal, const double weight = 1.) { if (SLRSolvers.empty()) { @@ -240,16 +240,16 @@ struct TTransformationParameters { double FeatureNormalizer = 1.; Y_SAVELOAD_DEFINE(RegressionFactor, - RegressionIntercept, - FeatureOffset, - FeatureNormalizer); + RegressionIntercept, + FeatureOffset, + FeatureNormalizer); }; class TFeaturesTransformer { private: ETransformationType TransformationType; TTransformationParameters TransformationParameters; - + public: Y_SAVELOAD_DEFINE(TransformationType, TransformationParameters); @@ -264,10 +264,10 @@ public: double Transformation(const double value) const { switch (TransformationType) { - case ETransformationType::TT_IDENTITY: { + case ETransformationType::TT_IDENTITY: { return value; } - case ETransformationType::TT_SIGMA: { + case ETransformationType::TT_SIGMA: { const double valueWithoutOffset = value - TransformationParameters.FeatureOffset; const double transformedValue = valueWithoutOffset / (fabs(valueWithoutOffset) + TransformationParameters.FeatureNormalizer); return TransformationParameters.RegressionIntercept + TransformationParameters.RegressionFactor * transformedValue; @@ -290,7 +290,7 @@ private: ETransformationType TransformationType; TVector<TPoint> Points; - + public: TFeaturesTransformerLearner(const ETransformationType transformationType) : TransformationType(transformationType) @@ -317,7 +317,7 @@ private: THashMap<double, TBucket> Buckets; double Step; - + public: TFastFeaturesTransformerLearner(const ETransformationType transformationType, const double step = 0.1) : TransformationType(transformationType) diff --git a/library/cpp/linear_regression/linear_regression_ut.cpp b/library/cpp/linear_regression/linear_regression_ut.cpp index e71a16b67a..d6e179bc0c 100644 --- a/library/cpp/linear_regression/linear_regression_ut.cpp +++ b/library/cpp/linear_regression/linear_regression_ut.cpp @@ -3,7 +3,7 @@ #include <util/generic/vector.h> #include <util/generic/ymath.h> -#include <util/random/random.h> +#include <util/random/random.h> #include <util/system/defaults.h> @@ -200,14 +200,14 @@ Y_UNIT_TEST_SUITE(TLinearRegressionTest) { } const double intercept = 10; - TVector<TVector<double>> featuresMatrix; + TVector<TVector<double>> featuresMatrix; TVector<double> goals; TVector<double> weights; for (size_t instanceNumber = 0; instanceNumber < instancesCount; ++instanceNumber) { TVector<double> features; for (size_t featureNumber = 0; featureNumber < featuresCount; ++featureNumber) { - features.push_back(RandomNumber<double>()); + features.push_back(RandomNumber<double>()); } featuresMatrix.push_back(features); diff --git a/library/cpp/linear_regression/welford.h b/library/cpp/linear_regression/welford.h index ee865d6693..89b5ce39ea 100644 --- a/library/cpp/linear_regression/welford.h +++ b/library/cpp/linear_regression/welford.h @@ -9,7 +9,7 @@ class TMeanCalculator { private: double Mean = 0.; TKahanAccumulator<double> SumWeights; - + public: Y_SAVELOAD_DEFINE(Mean, SumWeights); @@ -20,11 +20,11 @@ public: double GetSumWeights() const; void Reset(); - bool operator<(const TMeanCalculator& other) const { + bool operator<(const TMeanCalculator& other) const { return Mean < other.Mean; } - bool operator>(const TMeanCalculator& other) const { + bool operator>(const TMeanCalculator& other) const { return Mean > other.Mean; } }; @@ -38,7 +38,7 @@ private: double SecondValueMean = 0.; TKahanAccumulator<double> SumWeights; - + public: Y_SAVELOAD_DEFINE(Covariation, FirstValueMean, SecondValueMean, SumWeights); @@ -60,7 +60,7 @@ class TDeviationCalculator { private: double Deviation = 0.; TMeanCalculator MeanCalculator; - + public: Y_SAVELOAD_DEFINE(Deviation, MeanCalculator); diff --git a/library/cpp/logger/all.h b/library/cpp/logger/all.h index ee1666844e..ec4593eda8 100644 --- a/library/cpp/logger/all.h +++ b/library/cpp/logger/all.h @@ -1,9 +1,9 @@ #pragma once - -#include "log.h" -#include "null.h" -#include "file.h" -#include "stream.h" -#include "thread.h" -#include "system.h" + +#include "log.h" +#include "null.h" +#include "file.h" +#include "stream.h" +#include "thread.h" +#include "system.h" #include "sync_page_cache_file.h" diff --git a/library/cpp/logger/backend.cpp b/library/cpp/logger/backend.cpp index b26bf5e88c..8804c990f2 100644 --- a/library/cpp/logger/backend.cpp +++ b/library/cpp/logger/backend.cpp @@ -1,15 +1,15 @@ -#include "backend.h" +#include "backend.h" #include <util/generic/vector.h> #include <util/system/mutex.h> #include <util/generic/singleton.h> #include <util/generic/yexception.h> - + namespace { class TGlobalLogsStorage { private: TVector<TLogBackend*> Backends; TMutex Mutex; - + public: void Register(TLogBackend* backend) { TGuard<TMutex> g(Mutex); @@ -40,7 +40,7 @@ namespace { }; } -template <> +template <> class TSingletonTraits<TGlobalLogsStorage> { public: static const size_t Priority = 50; @@ -52,11 +52,11 @@ ELogPriority TLogBackend::FiltrationLevel() const { TLogBackend::TLogBackend() noexcept { Singleton<TGlobalLogsStorage>()->Register(this); -} - +} + TLogBackend::~TLogBackend() { Singleton<TGlobalLogsStorage>()->UnRegister(this); -} +} void TLogBackend::ReopenLogNoFlush() { ReopenLog(); diff --git a/library/cpp/logger/backend.h b/library/cpp/logger/backend.h index d088726d6d..20e01eabd1 100644 --- a/library/cpp/logger/backend.h +++ b/library/cpp/logger/backend.h @@ -5,17 +5,17 @@ #include <util/generic/noncopyable.h> #include <cstddef> - -struct TLogRecord; + +struct TLogRecord; // NOTE: be aware that all `TLogBackend`s are registred in singleton. -class TLogBackend: public TNonCopyable { -public: - TLogBackend() noexcept; - virtual ~TLogBackend(); - - virtual void WriteData(const TLogRecord& rec) = 0; - virtual void ReopenLog() = 0; +class TLogBackend: public TNonCopyable { +public: + TLogBackend() noexcept; + virtual ~TLogBackend(); + + virtual void WriteData(const TLogRecord& rec) = 0; + virtual void ReopenLog() = 0; // Does not guarantee consistency with previous WriteData() calls: // log entries could be written to the new (reopened) log file due to @@ -27,4 +27,4 @@ public: static void ReopenAllBackends(bool flush = true); virtual size_t QueueSize() const; -}; +}; diff --git a/library/cpp/logger/element.cpp b/library/cpp/logger/element.cpp index b510fe16e1..80a9d74419 100644 --- a/library/cpp/logger/element.cpp +++ b/library/cpp/logger/element.cpp @@ -1,15 +1,15 @@ -#include "log.h" -#include "element.h" - +#include "log.h" +#include "element.h" + #include <utility> TLogElement::TLogElement(const TLog* parent) - : Parent_(parent) - , Priority_(Parent_->DefaultPriority()) -{ - Reset(); -} - + : Parent_(parent) + , Priority_(Parent_->DefaultPriority()) +{ + Reset(); +} + TLogElement::TLogElement(const TLog* parent, ELogPriority priority) : Parent_(parent) , Priority_(priority) @@ -18,21 +18,21 @@ TLogElement::TLogElement(const TLog* parent, ELogPriority priority) } TLogElement::~TLogElement() { - try { - Finish(); - } catch (...) { - } -} - -void TLogElement::DoFlush() { + try { + Finish(); + } catch (...) { + } +} + +void TLogElement::DoFlush() { if (IsNull()) { return; } - const size_t filled = Filled(); - - if (filled) { - Parent_->Write(Priority_, Data(), filled); - Reset(); - } -} + const size_t filled = Filled(); + + if (filled) { + Parent_->Write(Priority_, Data(), filled); + Reset(); + } +} diff --git a/library/cpp/logger/element.h b/library/cpp/logger/element.h index fc9bff851f..73ce81bf2f 100644 --- a/library/cpp/logger/element.h +++ b/library/cpp/logger/element.h @@ -1,48 +1,48 @@ #pragma once - -#include "priority.h" - -#include <util/stream/tempbuf.h> - -class TLog; - -/* - * better do not use directly - */ + +#include "priority.h" + +#include <util/stream/tempbuf.h> + +class TLog; + +/* + * better do not use directly + */ class TLogElement: public TTempBufOutput { -public: - TLogElement(const TLog* parent); +public: + TLogElement(const TLog* parent); TLogElement(const TLog* parent, ELogPriority priority); - TLogElement(TLogElement&&) noexcept = default; - TLogElement& operator=(TLogElement&&) noexcept = default; - - ~TLogElement() override; + TLogElement(TLogElement&&) noexcept = default; + TLogElement& operator=(TLogElement&&) noexcept = default; - template <class T> - inline TLogElement& operator<<(const T& t) { + ~TLogElement() override; + + template <class T> + inline TLogElement& operator<<(const T& t) { static_cast<IOutputStream&>(*this) << t; - - return *this; - } - - /* - * for pretty usage: logger << TLOG_ERROR << "Error description"; - */ + + return *this; + } + + /* + * for pretty usage: logger << TLOG_ERROR << "Error description"; + */ inline TLogElement& operator<<(ELogPriority priority) { - Flush(); - Priority_ = priority; - return *this; - } - + Flush(); + Priority_ = priority; + return *this; + } + ELogPriority Priority() const noexcept { - return Priority_; - } - -protected: - void DoFlush() override; - -protected: - const TLog* Parent_; + return Priority_; + } + +protected: + void DoFlush() override; + +protected: + const TLog* Parent_; ELogPriority Priority_; -}; +}; diff --git a/library/cpp/logger/file.cpp b/library/cpp/logger/file.cpp index 15a4946eda..c5ecd5550c 100644 --- a/library/cpp/logger/file.cpp +++ b/library/cpp/logger/file.cpp @@ -1,59 +1,59 @@ -#include "file.h" -#include "record.h" - -#include <util/system/file.h> -#include <util/system/rwlock.h> +#include "file.h" +#include "record.h" + +#include <util/system/file.h> +#include <util/system/rwlock.h> #include <util/generic/string.h> - -/* - * file log - */ + +/* + * file log + */ class TFileLogBackend::TImpl { -public: +public: inline TImpl(const TString& path) - : File_(OpenFile(path)) - { - } - - inline void WriteData(const TLogRecord& rec) { - //many writes are thread-safe - TReadGuard guard(Lock_); - - File_.Write(rec.Data, rec.Len); - } - - inline void ReopenLog() { - //but log rotate not thread-safe - TWriteGuard guard(Lock_); - - File_.LinkTo(OpenFile(File_.GetName())); - } - -private: + : File_(OpenFile(path)) + { + } + + inline void WriteData(const TLogRecord& rec) { + //many writes are thread-safe + TReadGuard guard(Lock_); + + File_.Write(rec.Data, rec.Len); + } + + inline void ReopenLog() { + //but log rotate not thread-safe + TWriteGuard guard(Lock_); + + File_.LinkTo(OpenFile(File_.GetName())); + } + +private: static inline TFile OpenFile(const TString& path) { - return TFile(path, OpenAlways | WrOnly | ForAppend | Seq | NoReuse); - } - -private: - TRWMutex Lock_; - TFile File_; -}; - + return TFile(path, OpenAlways | WrOnly | ForAppend | Seq | NoReuse); + } + +private: + TRWMutex Lock_; + TFile File_; +}; + TFileLogBackend::TFileLogBackend(const TString& path) - : Impl_(new TImpl(path)) -{ -} - + : Impl_(new TImpl(path)) +{ +} + TFileLogBackend::~TFileLogBackend() { -} - -void TFileLogBackend::WriteData(const TLogRecord& rec) { - Impl_->WriteData(rec); -} - -void TFileLogBackend::ReopenLog() { +} + +void TFileLogBackend::WriteData(const TLogRecord& rec) { + Impl_->WriteData(rec); +} + +void TFileLogBackend::ReopenLog() { TAtomicSharedPtr<TImpl> copy = Impl_; if (copy) { copy->ReopenLog(); } -} +} diff --git a/library/cpp/logger/file.h b/library/cpp/logger/file.h index 10b4cd0c20..fb8d3879c4 100644 --- a/library/cpp/logger/file.h +++ b/library/cpp/logger/file.h @@ -1,19 +1,19 @@ #pragma once -#include "backend.h" - +#include "backend.h" + #include <util/generic/fwd.h> #include <util/generic/ptr.h> - -class TFileLogBackend: public TLogBackend { -public: + +class TFileLogBackend: public TLogBackend { +public: TFileLogBackend(const TString& path); - ~TFileLogBackend() override; - - void WriteData(const TLogRecord& rec) override; - void ReopenLog() override; - -private: - class TImpl; + ~TFileLogBackend() override; + + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override; + +private: + class TImpl; TAtomicSharedPtr<TImpl> Impl_; }; diff --git a/library/cpp/logger/filter.h b/library/cpp/logger/filter.h index 9ef83fb58c..5355fa4493 100644 --- a/library/cpp/logger/filter.h +++ b/library/cpp/logger/filter.h @@ -1,5 +1,5 @@ #pragma once - + #include "priority.h" #include "record.h" #include "backend.h" @@ -8,7 +8,7 @@ class TFilteredLogBackend: public TLogBackend { THolder<TLogBackend> Backend; ELogPriority Level; - + public: TFilteredLogBackend(THolder<TLogBackend>&& t, ELogPriority level = LOG_MAX_PRIORITY) noexcept : Backend(std::move(t)) diff --git a/library/cpp/logger/global/common.h b/library/cpp/logger/global/common.h index 7dcf650dec..0f5b9893c6 100644 --- a/library/cpp/logger/global/common.h +++ b/library/cpp/logger/global/common.h @@ -36,7 +36,7 @@ namespace NLoggingImpl { public: inline static bool Usage() { return SingletonWithPriority<TPtr, SingletonPriority>()->Instance.Get(); - } + } inline static T* Get() { return SingletonWithPriority<TPtr, SingletonPriority>()->Instance.Get(); @@ -139,11 +139,11 @@ namespace NPrivateGlobalLogger { return true; } }; -} +} #define LOGGER_GENERIC_LOG_CHECKED(logger, preprocessor, level, message) (*GetLoggerForce<preprocessor>(logger, TLogRecordContext(__LOCATION__, message, level))) -#define LOGGER_CHECKED_GENERIC_LOG(logger, preprocessor, level, message) \ - (preprocessor::CheckLoggingContext(logger, TLogRecordContext(__LOCATION__, message, level))) && NPrivateGlobalLogger::TEatStream() | (*(preprocessor::StartRecord(logger, TLogRecordContext(__LOCATION__, message, level), nullptr))) +#define LOGGER_CHECKED_GENERIC_LOG(logger, preprocessor, level, message) \ + (preprocessor::CheckLoggingContext(logger, TLogRecordContext(__LOCATION__, message, level))) && NPrivateGlobalLogger::TEatStream() | (*(preprocessor::StartRecord(logger, TLogRecordContext(__LOCATION__, message, level), nullptr))) #define SINGLETON_GENERIC_LOG_CHECKED(type, preprocessor, level, message) LOGGER_GENERIC_LOG_CHECKED(TLoggerOperator<type>::Log(), preprocessor, level, message) #define SINGLETON_CHECKED_GENERIC_LOG(type, preprocessor, level, message) LOGGER_CHECKED_GENERIC_LOG(TLoggerOperator<type>::Log(), preprocessor, level, message) diff --git a/library/cpp/logger/global/global.cpp b/library/cpp/logger/global/global.cpp index 9fbd10f666..d871fcd350 100644 --- a/library/cpp/logger/global/global.cpp +++ b/library/cpp/logger/global/global.cpp @@ -26,12 +26,12 @@ bool GlobalLogInitialized() { return TLoggerOperator<TGlobalLog>::Usage(); } -template <> +template <> TGlobalLog* CreateDefaultLogger<TGlobalLog>() { return new TGlobalLog("console", TLOG_INFO); } -template <> +template <> TNullLog* CreateDefaultLogger<TNullLog>() { return new TNullLog("null"); } diff --git a/library/cpp/logger/global/global.h b/library/cpp/logger/global/global.h index cbe71b16ea..0613709ff0 100644 --- a/library/cpp/logger/global/global.h +++ b/library/cpp/logger/global/global.h @@ -30,7 +30,7 @@ public: } }; -template <> +template <> TGlobalLog* CreateDefaultLogger<TGlobalLog>(); class TNullLog: public TLog { @@ -46,7 +46,7 @@ public: } }; -template <> +template <> TNullLog* CreateDefaultLogger<TNullLog>(); template <> @@ -79,9 +79,9 @@ public: #define VERIFY_WITH_LOG(expr, msg, ...) \ do { \ - if (Y_UNLIKELY(!(expr))) { \ + if (Y_UNLIKELY(!(expr))) { \ FATAL_LOG << Sprintf(msg, ##__VA_ARGS__) << Endl; \ - Y_VERIFY(false, msg, ##__VA_ARGS__); \ + Y_VERIFY(false, msg, ##__VA_ARGS__); \ }; \ } while (0); @@ -90,7 +90,7 @@ namespace NPrivateGlobalLogger { public: ~TVerifyEvent(); template <class T> - inline TVerifyEvent& operator<<(const T& t) { + inline TVerifyEvent& operator<<(const T& t) { static_cast<IOutputStream&>(*this) << t; return *this; diff --git a/library/cpp/logger/global/rty_formater.cpp b/library/cpp/logger/global/rty_formater.cpp index 305f8470c5..c7799280bd 100644 --- a/library/cpp/logger/global/rty_formater.cpp +++ b/library/cpp/logger/global/rty_formater.cpp @@ -18,7 +18,7 @@ namespace { instant.LocalTime(&tm); // both stftime and sprintf exclude the terminating null byte from the return value - char* pos = begin; + char* pos = begin; pos += strftime(pos, end - pos, "%Y-%m-%d %H:%M:%S.", &tm); pos += sprintf(pos, "%03" PRIu32, instant.MilliSecondsOfSecond()); pos += strftime(pos, end - pos, " %z", &tm); @@ -28,7 +28,7 @@ namespace { } namespace NLoggingImpl { - IOutputStream& operator<<(IOutputStream& out, TLocalTimeS localTimeS) { + IOutputStream& operator<<(IOutputStream& out, TLocalTimeS localTimeS) { char buffer[LocalTimeSBufferSize]; size_t len = PrintLocalTimeS(localTimeS.GetInstant(), buffer, buffer + sizeof(buffer)); out.Write(buffer, len); diff --git a/library/cpp/logger/global/rty_formater.h b/library/cpp/logger/global/rty_formater.h index 6532e1d769..cf58bb7949 100644 --- a/library/cpp/logger/global/rty_formater.h +++ b/library/cpp/logger/global/rty_formater.h @@ -1,5 +1,5 @@ #pragma once - + #include "common.h" namespace NMemInfo { diff --git a/library/cpp/logger/global/rty_formater_ut.cpp b/library/cpp/logger/global/rty_formater_ut.cpp index 551a97c5bf..c5d4461ae7 100644 --- a/library/cpp/logger/global/rty_formater_ut.cpp +++ b/library/cpp/logger/global/rty_formater_ut.cpp @@ -26,6 +26,6 @@ Y_UNIT_TEST_SUITE(NLoggingImplTest) { Y_UNIT_TEST(TestTLocalTimeSAddRight) { NLoggingImpl::TLocalTimeS lt(TInstant::ParseIso8601Deprecated(SampleISO8601)); TString prefix("prefix"); - UNIT_ASSERT_EQUAL(prefix + lt, prefix + SampleRtyLog); + UNIT_ASSERT_EQUAL(prefix + lt, prefix + SampleRtyLog); } } diff --git a/library/cpp/logger/log.cpp b/library/cpp/logger/log.cpp index e1d70cc3d2..33cfff57c0 100644 --- a/library/cpp/logger/log.cpp +++ b/library/cpp/logger/log.cpp @@ -2,16 +2,16 @@ #include "uninitialized_creator.h" #include "filter.h" #include "null.h" -#include "stream.h" +#include "stream.h" #include "thread.h" - -#include <util/string/cast.h> -#include <util/stream/printf.h> -#include <util/system/yassert.h> + +#include <util/string/cast.h> +#include <util/stream/printf.h> +#include <util/system/yassert.h> #include <util/generic/string.h> -#include <util/generic/scope.h> -#include <util/generic/yexception.h> - +#include <util/generic/scope.h> +#include <util/generic/yexception.h> + THolder<TLogBackend> CreateLogBackend(const TString& fname, ELogPriority priority, bool threaded) { TLogBackendCreatorUninitialized creator; creator.InitCustom(fname, priority, threaded); @@ -26,38 +26,38 @@ THolder<TOwningThreadedLogBackend> CreateOwningThreadedLogBackend(const TString& return MakeHolder<TOwningThreadedLogBackend>(CreateLogBackend(fname, LOG_MAX_PRIORITY, false).Release(), queueLen); } -class TLog::TImpl: public TAtomicRefCount<TImpl> { +class TLog::TImpl: public TAtomicRefCount<TImpl> { class TPriorityLogStream final: public IOutputStream { - public: + public: inline TPriorityLogStream(ELogPriority p, const TImpl* parent) - : Priority_(p) - , Parent_(parent) - { - } - - void DoWrite(const void* buf, size_t len) override { - Parent_->WriteData(Priority_, (const char*)buf, len); - } - - private: + : Priority_(p) + , Parent_(parent) + { + } + + void DoWrite(const void* buf, size_t len) override { + Parent_->WriteData(Priority_, (const char*)buf, len); + } + + private: ELogPriority Priority_ = LOG_DEF_PRIORITY; const TImpl* Parent_ = nullptr; - }; - -public: + }; + +public: inline TImpl(THolder<TLogBackend> backend) : Backend_(std::move(backend)) - { - } - - inline void ReopenLog() { - if (!IsOpen()) { - return; - } - + { + } + + inline void ReopenLog() { + if (!IsOpen()) { + return; + } + Backend_->ReopenLog(); - } - + } + inline void ReopenLogNoFlush() { if (!IsOpen()) { return; @@ -67,133 +67,133 @@ public: } inline void AddLog(ELogPriority priority, const char* format, va_list args) const { - if (!IsOpen()) { - return; + if (!IsOpen()) { + return; } - TPriorityLogStream ls(priority, this); - - Printf(ls, format, args); - } + TPriorityLogStream ls(priority, this); + Printf(ls, format, args); + } + inline void ResetBackend(THolder<TLogBackend> backend) noexcept { Backend_ = std::move(backend); - } - + } + inline THolder<TLogBackend> ReleaseBackend() noexcept { return std::move(Backend_); - } - - inline bool IsNullLog() const noexcept { + } + + inline bool IsNullLog() const noexcept { return !IsOpen() || (dynamic_cast<TNullLogBackend*>(Backend_.Get()) != nullptr); - } - - inline bool IsOpen() const noexcept { + } + + inline bool IsOpen() const noexcept { return nullptr != Backend_.Get(); - } + } - inline void CloseLog() noexcept { + inline void CloseLog() noexcept { Backend_.Destroy(); - Y_ASSERT(!IsOpen()); - } - + Y_ASSERT(!IsOpen()); + } + inline void WriteData(ELogPriority priority, const char* data, size_t len) const { - if (IsOpen()) { + if (IsOpen()) { Backend_->WriteData(TLogRecord(priority, data, len)); } - } + } inline ELogPriority DefaultPriority() noexcept { - return DefaultPriority_; - } + return DefaultPriority_; + } inline void SetDefaultPriority(ELogPriority priority) noexcept { - DefaultPriority_ = priority; - } - + DefaultPriority_ = priority; + } + inline ELogPriority FiltrationLevel() const noexcept { return Backend_->FiltrationLevel(); - } - + } + inline size_t BackEndQueueSize() const { return Backend_->QueueSize(); } -private: +private: THolder<TLogBackend> Backend_; ELogPriority DefaultPriority_ = LOG_DEF_PRIORITY; -}; - -TLog::TLog() +}; + +TLog::TLog() : Impl_(MakeIntrusive<TImpl>(nullptr)) -{ -} - +{ +} + TLog::TLog(const TString& fname, ELogPriority priority) : TLog(CreateLogBackend(fname, priority, false)) { -} - +} + TLog::TLog(THolder<TLogBackend> backend) : Impl_(MakeIntrusive<TImpl>(std::move(backend))) -{ -} - +{ +} + TLog::TLog(const TLog&) = default; TLog::TLog(TLog&&) = default; TLog::~TLog() = default; TLog& TLog::operator=(const TLog&) = default; TLog& TLog::operator=(TLog&&) = default; - + bool TLog::IsOpen() const noexcept { - return Impl_->IsOpen(); -} - + return Impl_->IsOpen(); +} + void TLog::AddLog(const char* format, ...) const { - va_list args; - va_start(args, format); - - Y_DEFER { - va_end(args); - }; - - Impl_->AddLog(Impl_->DefaultPriority(), format, args); -} - + va_list args; + va_start(args, format); + + Y_DEFER { + va_end(args); + }; + + Impl_->AddLog(Impl_->DefaultPriority(), format, args); +} + void TLog::AddLog(ELogPriority priority, const char* format, ...) const { va_list args; va_start(args, format); - Y_DEFER { + Y_DEFER { va_end(args); - }; + }; - Impl_->AddLog(priority, format, args); + Impl_->AddLog(priority, format, args); } void TLog::AddLogVAList(const char* format, va_list lst) { - Impl_->AddLog(Impl_->DefaultPriority(), format, lst); -} - -void TLog::ReopenLog() { + Impl_->AddLog(Impl_->DefaultPriority(), format, lst); +} + +void TLog::ReopenLog() { if (const auto copy = Impl_) { copy->ReopenLog(); } -} - +} + void TLog::ReopenLogNoFlush() { if (const auto copy = Impl_) { copy->ReopenLogNoFlush(); } } -void TLog::CloseLog() { - Impl_->CloseLog(); -} - +void TLog::CloseLog() { + Impl_->CloseLog(); +} + void TLog::SetDefaultPriority(ELogPriority priority) noexcept { - Impl_->SetDefaultPriority(priority); + Impl_->SetDefaultPriority(priority); } ELogPriority TLog::FiltrationLevel() const noexcept { @@ -201,23 +201,23 @@ ELogPriority TLog::FiltrationLevel() const noexcept { } ELogPriority TLog::DefaultPriority() const noexcept { - return Impl_->DefaultPriority(); -} - + return Impl_->DefaultPriority(); +} + bool TLog::OpenLog(const char* path, ELogPriority lp) { - if (path) { + if (path) { ResetBackend(CreateLogBackend(path, lp)); - } else { + } else { ResetBackend(MakeHolder<TStreamLogBackend>(&Cerr)); - } - - return true; -} - + } + + return true; +} + void TLog::ResetBackend(THolder<TLogBackend> backend) noexcept { Impl_->ResetBackend(std::move(backend)); -} - +} + bool TLog::IsNullLog() const noexcept { return Impl_->IsNullLog(); } @@ -241,7 +241,7 @@ void TLog::Write(ELogPriority priority, const TStringBuf data) const { void TLog::Write(const char* data, size_t len) const { Write(Impl_->DefaultPriority(), data, len); -} +} void TLog::SetFormatter(TLogFormatter formatter) noexcept { Formatter_ = std::move(formatter); diff --git a/library/cpp/logger/log.h b/library/cpp/logger/log.h index 8be984ccc8..7cb2a471f1 100644 --- a/library/cpp/logger/log.h +++ b/library/cpp/logger/log.h @@ -1,17 +1,17 @@ #pragma once #include "backend.h" -#include "element.h" -#include "priority.h" +#include "element.h" +#include "priority.h" #include "record.h" #include "thread.h" - + #include <util/generic/fwd.h> -#include <util/generic/ptr.h> +#include <util/generic/ptr.h> #include <functional> -#include <cstdarg> - +#include <cstdarg> + using TLogFormatter = std::function<TString(ELogPriority priority, TStringBuf)>; // Logging facilities interface. @@ -30,21 +30,21 @@ using TLogFormatter = std::function<TString(ELogPriority priority, TStringBuf)>; // ``` // // Users are encouraged to copy `TLog` instance. -class TLog { -public: +class TLog { +public: // Construct empty logger all writes will be spilled. - TLog(); + TLog(); // Construct file logger. TLog(const TString& fname, ELogPriority priority = LOG_MAX_PRIORITY); // Construct any type of logger TLog(THolder<TLogBackend> backend); - + TLog(const TLog&); TLog(TLog&&); - ~TLog(); + ~TLog(); TLog& operator=(const TLog&); TLog& operator=(TLog&&); - + // Change underlying backend. // NOTE: not thread safe. void ResetBackend(THolder<TLogBackend> backend) noexcept; @@ -53,30 +53,30 @@ public: THolder<TLogBackend> ReleaseBackend() noexcept; // Check if underlying backend is defined and is not null. // NOTE: not thread safe with respect to `ResetBackend` and `ReleaseBackend`. - bool IsNullLog() const noexcept; - + bool IsNullLog() const noexcept; + // Write message to the log. // // @param[in] priority Message priority to use. // @param[in] message Message to write. void Write(ELogPriority priority, TStringBuf message) const; // Write message to the log using `DefaultPriority()`. - void Write(const char* data, size_t len) const; + void Write(const char* data, size_t len) const; // Write message to the log, but pass the message in a c-style. void Write(ELogPriority priority, const char* data, size_t len) const; // Write message to the log in a c-like printf style. void Y_PRINTF_FORMAT(3, 4) AddLog(ELogPriority priority, const char* format, ...) const; // Write message to the log in a c-like printf style with `DefaultPriority()` priority. - void Y_PRINTF_FORMAT(2, 3) AddLog(const char* format, ...) const; + void Y_PRINTF_FORMAT(2, 3) AddLog(const char* format, ...) const; // Call `ReopenLog()` of the underlying backend. - void ReopenLog(); + void ReopenLog(); // Call `ReopenLogNoFlush()` of the underlying backend. void ReopenLogNoFlush(); // Call `QueueSize()` of the underlying backend. size_t BackEndQueueSize() const; - + // Set log default priority. // NOTE: not thread safe. void SetDefaultPriority(ELogPriority priority) noexcept; @@ -89,13 +89,13 @@ public: // Set current log formatter. void SetFormatter(TLogFormatter formatter) noexcept; - template <class T> + template <class T> inline TLogElement operator<<(const T& t) const { - TLogElement ret(this); - ret << t; - return ret; - } - + TLogElement ret(this); + ret << t; + return ret; + } + public: // These methods are deprecated and present here only for compatibility reasons (for 13 years // already ...). Do not use them. @@ -104,11 +104,11 @@ public: void AddLogVAList(const char* format, va_list lst); void CloseLog(); -private: - class TImpl; +private: + class TImpl; TIntrusivePtr<TImpl> Impl_; TLogFormatter Formatter_; -}; +}; THolder<TLogBackend> CreateLogBackend(const TString& fname, ELogPriority priority = LOG_MAX_PRIORITY, bool threaded = false); THolder<TLogBackend> CreateFilteredOwningThreadedLogBackend(const TString& fname, ELogPriority priority = LOG_MAX_PRIORITY, size_t queueLen = 0); diff --git a/library/cpp/logger/log_ut.cpp b/library/cpp/logger/log_ut.cpp index 8de46f17f5..498a667756 100644 --- a/library/cpp/logger/log_ut.cpp +++ b/library/cpp/logger/log_ut.cpp @@ -1,83 +1,83 @@ -#include "all.h" - +#include "all.h" + #include <library/cpp/testing/unittest/registar.h> - -#include <util/system/fs.h> + +#include <util/system/fs.h> #include <util/system/rwlock.h> #include <util/system/yield.h> #include <util/memory/blob.h> -#include <util/stream/file.h> +#include <util/stream/file.h> #include <util/generic/string.h> #include <util/generic/vector.h> - -class TLogTest: public TTestBase { - UNIT_TEST_SUITE(TLogTest); - UNIT_TEST(TestFile) - UNIT_TEST(TestFormat) - UNIT_TEST(TestWrite) - UNIT_TEST(TestThreaded) + +class TLogTest: public TTestBase { + UNIT_TEST_SUITE(TLogTest); + UNIT_TEST(TestFile) + UNIT_TEST(TestFormat) + UNIT_TEST(TestWrite) + UNIT_TEST(TestThreaded) UNIT_TEST(TestThreadedWithOverflow) UNIT_TEST(TestNoFlush) - UNIT_TEST_SUITE_END(); - -private: - void TestFile(); - void TestFormat(); - void TestWrite(); - void TestThreaded(); + UNIT_TEST_SUITE_END(); + +private: + void TestFile(); + void TestFormat(); + void TestWrite(); + void TestThreaded(); void TestThreadedWithOverflow(); void TestNoFlush(); - void SetUp() override; - void TearDown() override; -}; - -UNIT_TEST_SUITE_REGISTRATION(TLogTest); - -#define LOGFILE "tmplogfile" - -void TLogTest::TestFile() { - { - TLog log; - - { - TLog filelog(LOGFILE); - - log = filelog; - } - - int v1 = 12; - unsigned v2 = 34; - double v3 = 3.0; - const char* v4 = "qwqwqw"; - - log.ReopenLog(); - log.AddLog("some useful data %d, %u, %lf, %s\n", v1, v2, v3, v4); - } - + void SetUp() override; + void TearDown() override; +}; + +UNIT_TEST_SUITE_REGISTRATION(TLogTest); + +#define LOGFILE "tmplogfile" + +void TLogTest::TestFile() { + { + TLog log; + + { + TLog filelog(LOGFILE); + + log = filelog; + } + + int v1 = 12; + unsigned v2 = 34; + double v3 = 3.0; + const char* v4 = "qwqwqw"; + + log.ReopenLog(); + log.AddLog("some useful data %d, %u, %lf, %s\n", v1, v2, v3, v4); + } + TBlob data = TBlob::FromFileSingleThreaded(LOGFILE); - + UNIT_ASSERT_EQUAL(TString((const char*)data.Begin(), data.Size()), "some useful data 12, 34, 3.000000, qwqwqw\n"); -} - -void TLogTest::TestThreaded() { - { - TFileLogBackend fb(LOGFILE); +} + +void TLogTest::TestThreaded() { + { + TFileLogBackend fb(LOGFILE); TLog log(THolder(new TThreadedLogBackend(&fb))); - - int v1 = 12; - unsigned v2 = 34; - double v3 = 3.0; - const char* v4 = "qwqwqw"; - - log.ReopenLog(); - log.AddLog("some useful data %d, %u, %lf, %s\n", v1, v2, v3, v4); - } - + + int v1 = 12; + unsigned v2 = 34; + double v3 = 3.0; + const char* v4 = "qwqwqw"; + + log.ReopenLog(); + log.AddLog("some useful data %d, %u, %lf, %s\n", v1, v2, v3, v4); + } + TBlob data = TBlob::FromFileSingleThreaded(LOGFILE); - + UNIT_ASSERT_EQUAL(TString((const char*)data.Begin(), data.Size()), "some useful data 12, 34, 3.000000, qwqwqw\n"); -} - +} + void TLogTest::TestThreadedWithOverflow() { class TFakeLogBackend: public TLogBackend { public: @@ -150,42 +150,42 @@ void TLogTest::TestNoFlush() { UNIT_ASSERT_EQUAL(TString((const char*)data.Begin(), data.Size()), "some useful data 12, 34, 3.000000, qwqwqw\n"); } -void TLogTest::TestFormat() { - TStringStream data; - - { +void TLogTest::TestFormat() { + TStringStream data; + + { TLog log(THolder(new TStreamLogBackend(&data))); - - log << "qw" - << " " - << "1234" << 1234 << " " << 12.3 << 'q' << Endl; - } - - UNIT_ASSERT_EQUAL(data.Str(), "qw 12341234 12.3q\n"); -} - -void TLogTest::TestWrite() { - TStringStream data; + + log << "qw" + << " " + << "1234" << 1234 << " " << 12.3 << 'q' << Endl; + } + + UNIT_ASSERT_EQUAL(data.Str(), "qw 12341234 12.3q\n"); +} + +void TLogTest::TestWrite() { + TStringStream data; TString test; - - { + + { TLog log(THolder(new TStreamLogBackend(&data))); - - for (size_t i = 0; i < 1000; ++i) { + + for (size_t i = 0; i < 1000; ++i) { TVector<char> buf(i, (char)i); - + test.append(buf.data(), buf.size()); log.Write(buf.data(), buf.size()); - } - } - - UNIT_ASSERT_EQUAL(data.Str(), test); -} - -void TLogTest::SetUp() { - TearDown(); -} - -void TLogTest::TearDown() { - NFs::Remove(LOGFILE); -} + } + } + + UNIT_ASSERT_EQUAL(data.Str(), test); +} + +void TLogTest::SetUp() { + TearDown(); +} + +void TLogTest::TearDown() { + NFs::Remove(LOGFILE); +} diff --git a/library/cpp/logger/null.cpp b/library/cpp/logger/null.cpp index debb22f794..9ed26c35ae 100644 --- a/library/cpp/logger/null.cpp +++ b/library/cpp/logger/null.cpp @@ -1,13 +1,13 @@ -#include "null.h" - -TNullLogBackend::TNullLogBackend() { -} - +#include "null.h" + +TNullLogBackend::TNullLogBackend() { +} + TNullLogBackend::~TNullLogBackend() { -} - -void TNullLogBackend::WriteData(const TLogRecord&) { -} - -void TNullLogBackend::ReopenLog() { -} +} + +void TNullLogBackend::WriteData(const TLogRecord&) { +} + +void TNullLogBackend::ReopenLog() { +} diff --git a/library/cpp/logger/null.h b/library/cpp/logger/null.h index a02f250b00..277d4b583f 100644 --- a/library/cpp/logger/null.h +++ b/library/cpp/logger/null.h @@ -1,12 +1,12 @@ #pragma once -#include "backend.h" +#include "backend.h" -class TNullLogBackend: public TLogBackend { -public: - TNullLogBackend(); - ~TNullLogBackend() override; - - void WriteData(const TLogRecord& rec) override; - void ReopenLog() override; -}; +class TNullLogBackend: public TLogBackend { +public: + TNullLogBackend(); + ~TNullLogBackend() override; + + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override; +}; diff --git a/library/cpp/logger/priority.h b/library/cpp/logger/priority.h index d2a9fa0a07..db7607ef22 100644 --- a/library/cpp/logger/priority.h +++ b/library/cpp/logger/priority.h @@ -1,15 +1,15 @@ #pragma once enum ELogPriority { - TLOG_EMERG = 0 /* "EMERG" */, - TLOG_ALERT = 1 /* "ALERT" */, - TLOG_CRIT = 2 /* "CRITICAL_INFO" */, - TLOG_ERR = 3 /* "ERROR" */, - TLOG_WARNING = 4 /* "WARNING" */, - TLOG_NOTICE = 5 /* "NOTICE" */, - TLOG_INFO = 6 /* "INFO" */, - TLOG_DEBUG = 7 /* "DEBUG" */, - TLOG_RESOURCES = 8 /* "RESOURCES" */ + TLOG_EMERG = 0 /* "EMERG" */, + TLOG_ALERT = 1 /* "ALERT" */, + TLOG_CRIT = 2 /* "CRITICAL_INFO" */, + TLOG_ERR = 3 /* "ERROR" */, + TLOG_WARNING = 4 /* "WARNING" */, + TLOG_NOTICE = 5 /* "NOTICE" */, + TLOG_INFO = 6 /* "INFO" */, + TLOG_DEBUG = 7 /* "DEBUG" */, + TLOG_RESOURCES = 8 /* "RESOURCES" */ }; #define LOG_MAX_PRIORITY TLOG_RESOURCES -#define LOG_DEF_PRIORITY TLOG_INFO +#define LOG_DEF_PRIORITY TLOG_INFO diff --git a/library/cpp/logger/record.h b/library/cpp/logger/record.h index c28a7785fd..1b0f0a9774 100644 --- a/library/cpp/logger/record.h +++ b/library/cpp/logger/record.h @@ -1,18 +1,18 @@ #pragma once -#include "priority.h" - -#include <util/system/defaults.h> - -struct TLogRecord { - const char* Data; - size_t Len; +#include "priority.h" + +#include <util/system/defaults.h> + +struct TLogRecord { + const char* Data; + size_t Len; ELogPriority Priority; inline TLogRecord(ELogPriority priority, const char* data, size_t len) noexcept - : Data(data) - , Len(len) - , Priority(priority) - { - } -}; + : Data(data) + , Len(len) + , Priority(priority) + { + } +}; diff --git a/library/cpp/logger/stream.cpp b/library/cpp/logger/stream.cpp index 96787ad94b..5cd114bc83 100644 --- a/library/cpp/logger/stream.cpp +++ b/library/cpp/logger/stream.cpp @@ -1,19 +1,19 @@ -#include "stream.h" -#include "record.h" - -#include <util/stream/output.h> - +#include "stream.h" +#include "record.h" + +#include <util/stream/output.h> + TStreamLogBackend::TStreamLogBackend(IOutputStream* slave) - : Slave_(slave) -{ -} - + : Slave_(slave) +{ +} + TStreamLogBackend::~TStreamLogBackend() { -} - -void TStreamLogBackend::WriteData(const TLogRecord& rec) { - Slave_->Write(rec.Data, rec.Len); -} - -void TStreamLogBackend::ReopenLog() { -} +} + +void TStreamLogBackend::WriteData(const TLogRecord& rec) { + Slave_->Write(rec.Data, rec.Len); +} + +void TStreamLogBackend::ReopenLog() { +} diff --git a/library/cpp/logger/stream.h b/library/cpp/logger/stream.h index feb240afcb..bb89011919 100644 --- a/library/cpp/logger/stream.h +++ b/library/cpp/logger/stream.h @@ -1,17 +1,17 @@ #pragma once -#include "backend.h" +#include "backend.h" class IOutputStream; - -class TStreamLogBackend: public TLogBackend { -public: + +class TStreamLogBackend: public TLogBackend { +public: TStreamLogBackend(IOutputStream* slave); - ~TStreamLogBackend() override; - - void WriteData(const TLogRecord& rec) override; - void ReopenLog() override; - -private: + ~TStreamLogBackend() override; + + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override; + +private: IOutputStream* Slave_; -}; +}; diff --git a/library/cpp/logger/system.cpp b/library/cpp/logger/system.cpp index 42233f63d2..ad4ebef1c8 100644 --- a/library/cpp/logger/system.cpp +++ b/library/cpp/logger/system.cpp @@ -1,59 +1,59 @@ #include <util/stream/output.h> #include <util/stream/null.h> -#include <util/system/compat.h> -#include <util/system/yassert.h> -#include <util/system/defaults.h> -#include <util/generic/singleton.h> +#include <util/system/compat.h> +#include <util/system/yassert.h> +#include <util/system/defaults.h> +#include <util/generic/singleton.h> #include <util/generic/utility.h> - -#if defined(_unix_) -#include <syslog.h> -#endif - -#include "system.h" -#include "record.h" -#include "stream.h" - + +#if defined(_unix_) +#include <syslog.h> +#endif + +#include "system.h" +#include "record.h" +#include "stream.h" + TSysLogBackend::TSysLogBackend(const char* ident, EFacility facility, int flags) : Ident(ident) , Facility(facility) , Flags(flags) { -#if defined(_unix_) +#if defined(_unix_) Y_ASSERT(TSYSLOG_LOCAL0 <= facility && facility <= TSYSLOG_LOCAL7); - + static const int f2sf[] = { - LOG_LOCAL0, - LOG_LOCAL1, - LOG_LOCAL2, - LOG_LOCAL3, - LOG_LOCAL4, - LOG_LOCAL5, - LOG_LOCAL6, - LOG_LOCAL7}; - + LOG_LOCAL0, + LOG_LOCAL1, + LOG_LOCAL2, + LOG_LOCAL3, + LOG_LOCAL4, + LOG_LOCAL5, + LOG_LOCAL6, + LOG_LOCAL7}; + int sysflags = LOG_NDELAY | LOG_PID; - - if (flags & LogPerror) { + + if (flags & LogPerror) { sysflags |= LOG_PERROR; - } - - if (flags & LogCons) { + } + + if (flags & LogCons) { sysflags |= LOG_CONS; - } - + } + openlog(Ident.data(), sysflags, f2sf[(size_t)facility]); #endif } TSysLogBackend::~TSysLogBackend() { -#if defined(_unix_) +#if defined(_unix_) closelog(); #endif } -void TSysLogBackend::WriteData(const TLogRecord& rec) { -#if defined(_unix_) +void TSysLogBackend::WriteData(const TLogRecord& rec) { +#if defined(_unix_) syslog(ELogPriority2SyslogPriority(rec.Priority), "%.*s", (int)rec.Len, rec.Data); #else Y_UNUSED(rec); @@ -71,17 +71,17 @@ int TSysLogBackend::ELogPriority2SyslogPriority(ELogPriority priority) { return int(priority); #endif } - -namespace { - class TSysLogInstance: public TLog { - public: - inline TSysLogInstance() + +namespace { + class TSysLogInstance: public TLog { + public: + inline TSysLogInstance() : TLog(MakeHolder<TStreamLogBackend>(&Cnull)) - { - } - }; -} - -TLog& SysLogInstance() { - return *Singleton<TSysLogInstance>(); -} + { + } + }; +} + +TLog& SysLogInstance() { + return *Singleton<TSysLogInstance>(); +} diff --git a/library/cpp/logger/system.h b/library/cpp/logger/system.h index b8c60b3023..a4e38dece8 100644 --- a/library/cpp/logger/system.h +++ b/library/cpp/logger/system.h @@ -1,66 +1,66 @@ #pragma once -#include "log.h" -#include "backend.h" -#include "priority.h" - -#define YSYSLOG(priority, ...) SysLogInstance().AddLog((priority), __VA_ARGS__) -#define YSYSLOGINIT_FLAGS(ident, facility, flags) \ - struct TLogIniter { \ - TLogIniter() { \ +#include "log.h" +#include "backend.h" +#include "priority.h" + +#define YSYSLOG(priority, ...) SysLogInstance().AddLog((priority), __VA_ARGS__) +#define YSYSLOGINIT_FLAGS(ident, facility, flags) \ + struct TLogIniter { \ + TLogIniter() { \ SysLogInstance().ResetBackend(THolder<TLogBackend>( \ (ident) ? (TLogBackend*)(new TSysLogBackend((ident), (facility), (flags))) : (TLogBackend*)(new TNullLogBackend())));\ - } \ + } \ } Y_CAT(loginit, __LINE__); -#define YSYSLOGINIT(ident, facility) YSYSLOGINIT_FLAGS((ident), (facility), 0) +#define YSYSLOGINIT(ident, facility) YSYSLOGINIT_FLAGS((ident), (facility), 0) class TSysLogBackend: public TLogBackend { -public: - enum EFacility { - TSYSLOG_LOCAL0 = 0, - TSYSLOG_LOCAL1 = 1, - TSYSLOG_LOCAL2 = 2, - TSYSLOG_LOCAL3 = 3, - TSYSLOG_LOCAL4 = 4, - TSYSLOG_LOCAL5 = 5, - TSYSLOG_LOCAL6 = 6, - TSYSLOG_LOCAL7 = 7 - }; +public: + enum EFacility { + TSYSLOG_LOCAL0 = 0, + TSYSLOG_LOCAL1 = 1, + TSYSLOG_LOCAL2 = 2, + TSYSLOG_LOCAL3 = 3, + TSYSLOG_LOCAL4 = 4, + TSYSLOG_LOCAL5 = 5, + TSYSLOG_LOCAL6 = 6, + TSYSLOG_LOCAL7 = 7 + }; - enum EFlags { - LogPerror = 1, - LogCons = 2 - }; + enum EFlags { + LogPerror = 1, + LogCons = 2 + }; - TSysLogBackend(const char* ident, EFacility facility, int flags = 0); - ~TSysLogBackend() override; + TSysLogBackend(const char* ident, EFacility facility, int flags = 0); + ~TSysLogBackend() override; - void WriteData(const TLogRecord& rec) override; - void ReopenLog() override; + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override; virtual TString GetIdent() const { - return Ident; - } + return Ident; + } - virtual EFacility GetFacility() const { - return Facility; - } + virtual EFacility GetFacility() const { + return Facility; + } - virtual int GetFlags() const { - return Flags; - } + virtual int GetFlags() const { + return Flags; + } -protected: +protected: int ELogPriority2SyslogPriority(ELogPriority priority); TString Ident; - EFacility Facility; - int Flags; + EFacility Facility; + int Flags; }; -/* - * return system-wide logger instance - * better do not use in real programs(instead of robot, of course) - */ -TLog& SysLogInstance(); +/* + * return system-wide logger instance + * better do not use in real programs(instead of robot, of course) + */ +TLog& SysLogInstance(); diff --git a/library/cpp/logger/thread.cpp b/library/cpp/logger/thread.cpp index 0ccf9e374b..a2650f195a 100644 --- a/library/cpp/logger/thread.cpp +++ b/library/cpp/logger/thread.cpp @@ -1,77 +1,77 @@ -#include "thread.h" -#include "record.h" - +#include "thread.h" +#include "record.h" + #include <util/thread/pool.h> -#include <util/system/event.h> -#include <util/memory/addstorage.h> -#include <util/generic/ptr.h> -#include <util/generic/yexception.h> - -class TThreadedLogBackend::TImpl { - class TRec: public IObjectInQueue, public TAdditionalStorage<TRec>, public TLogRecord { - public: - inline TRec(TImpl* parent, const TLogRecord& rec) - : TLogRecord(rec.Priority, (const char*)AdditionalData(), rec.Len) - , Parent_(parent) - { - memcpy(AdditionalData(), rec.Data, rec.Len); - } - - inline ~TRec() override { - } - - private: - void Process(void* /*tsr*/) override { - THolder<TRec> This(this); - - Parent_->Slave_->WriteData(*this); - } - - private: - TImpl* Parent_; - }; - +#include <util/system/event.h> +#include <util/memory/addstorage.h> +#include <util/generic/ptr.h> +#include <util/generic/yexception.h> + +class TThreadedLogBackend::TImpl { + class TRec: public IObjectInQueue, public TAdditionalStorage<TRec>, public TLogRecord { + public: + inline TRec(TImpl* parent, const TLogRecord& rec) + : TLogRecord(rec.Priority, (const char*)AdditionalData(), rec.Len) + , Parent_(parent) + { + memcpy(AdditionalData(), rec.Data, rec.Len); + } + + inline ~TRec() override { + } + + private: + void Process(void* /*tsr*/) override { + THolder<TRec> This(this); + + Parent_->Slave_->WriteData(*this); + } + + private: + TImpl* Parent_; + }; + class TReopener: public IObjectInQueue, public TSystemEvent, public TAtomicRefCount<TReopener> { - public: - inline TReopener(TImpl* parent) - : Parent_(parent) - { - Ref(); - } - - inline ~TReopener() override { - } - - private: - void Process(void* /*tsr*/) override { - try { - Parent_->Slave_->ReopenLog(); - } catch (...) { - } - - Signal(); - UnRef(); - } - - private: - TImpl* Parent_; - }; - -public: + public: + inline TReopener(TImpl* parent) + : Parent_(parent) + { + Ref(); + } + + inline ~TReopener() override { + } + + private: + void Process(void* /*tsr*/) override { + try { + Parent_->Slave_->ReopenLog(); + } catch (...) { + } + + Signal(); + UnRef(); + } + + private: + TImpl* Parent_; + }; + +public: inline TImpl(TLogBackend* slave, size_t queuelen, std::function<void()> queueOverflowCallback = {}) - : Slave_(slave) + : Slave_(slave) , QueueOverflowCallback_(std::move(queueOverflowCallback)) - { - Queue_.Start(1, queuelen); - } - - inline ~TImpl() { - Queue_.Stop(); - } - - inline void WriteData(const TLogRecord& rec) { - THolder<TRec> obj(new (rec.Len) TRec(this, rec)); - + { + Queue_.Start(1, queuelen); + } + + inline ~TImpl() { + Queue_.Stop(); + } + + inline void WriteData(const TLogRecord& rec) { + THolder<TRec> obj(new (rec.Len) TRec(this, rec)); + if (Queue_.Add(obj.Get())) { Y_UNUSED(obj.Release()); return; @@ -80,28 +80,28 @@ public: if (QueueOverflowCallback_) { QueueOverflowCallback_(); } else { - ythrow yexception() << "log queue exhausted"; + ythrow yexception() << "log queue exhausted"; } - } - - // Write an emergency message when the memory allocator is corrupted. - // The TThreadedLogBackend object can't be used after this method is called. - inline void WriteEmergencyData(const TLogRecord& rec) noexcept { - Queue_.Stop(); - Slave_->WriteData(rec); - } - - inline void ReopenLog() { - TIntrusivePtr<TReopener> reopener(new TReopener(this)); - - if (!Queue_.Add(reopener.Get())) { - reopener->UnRef(); // Ref() was called in constructor - ythrow yexception() << "log queue exhausted"; - } - - reopener->Wait(); - } - + } + + // Write an emergency message when the memory allocator is corrupted. + // The TThreadedLogBackend object can't be used after this method is called. + inline void WriteEmergencyData(const TLogRecord& rec) noexcept { + Queue_.Stop(); + Slave_->WriteData(rec); + } + + inline void ReopenLog() { + TIntrusivePtr<TReopener> reopener(new TReopener(this)); + + if (!Queue_.Add(reopener.Get())) { + reopener->UnRef(); // Ref() was called in constructor + ythrow yexception() << "log queue exhausted"; + } + + reopener->Wait(); + } + inline void ReopenLogNoFlush() { Slave_->ReopenLogNoFlush(); } @@ -110,32 +110,32 @@ public: return Queue_.Size(); } -private: - TLogBackend* Slave_; +private: + TLogBackend* Slave_; TThreadPool Queue_{"ThreadedLogBack"}; const std::function<void()> QueueOverflowCallback_; -}; - -TThreadedLogBackend::TThreadedLogBackend(TLogBackend* slave) - : Impl_(new TImpl(slave, 0)) -{ -} - +}; + +TThreadedLogBackend::TThreadedLogBackend(TLogBackend* slave) + : Impl_(new TImpl(slave, 0)) +{ +} + TThreadedLogBackend::TThreadedLogBackend(TLogBackend* slave, size_t queuelen, std::function<void()> queueOverflowCallback) : Impl_(new TImpl(slave, queuelen, std::move(queueOverflowCallback))) -{ -} - +{ +} + TThreadedLogBackend::~TThreadedLogBackend() { -} - -void TThreadedLogBackend::WriteData(const TLogRecord& rec) { - Impl_->WriteData(rec); -} - -void TThreadedLogBackend::ReopenLog() { - Impl_->ReopenLog(); -} +} + +void TThreadedLogBackend::WriteData(const TLogRecord& rec) { + Impl_->WriteData(rec); +} + +void TThreadedLogBackend::ReopenLog() { + Impl_->ReopenLog(); +} void TThreadedLogBackend::ReopenLogNoFlush() { Impl_->ReopenLogNoFlush(); diff --git a/library/cpp/logger/thread.h b/library/cpp/logger/thread.h index 65f7a88e87..48cfe1349e 100644 --- a/library/cpp/logger/thread.h +++ b/library/cpp/logger/thread.h @@ -1,30 +1,30 @@ #pragma once -#include "backend.h" - +#include "backend.h" + #include <util/generic/ptr.h> - + #include <functional> -class TThreadedLogBackend: public TLogBackend { -public: - TThreadedLogBackend(TLogBackend* slave); +class TThreadedLogBackend: public TLogBackend { +public: + TThreadedLogBackend(TLogBackend* slave); TThreadedLogBackend(TLogBackend* slave, size_t queuelen, std::function<void()> queueOverflowCallback = {}); - ~TThreadedLogBackend() override; - - void WriteData(const TLogRecord& rec) override; - void ReopenLog() override; + ~TThreadedLogBackend() override; + + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override; void ReopenLogNoFlush() override; size_t QueueSize() const override; - - // Write an emergency message when the memory allocator is corrupted. - // The TThreadedLogBackend object can't be used after this method is called. - void WriteEmergencyData(const TLogRecord& rec); - -private: - class TImpl; - THolder<TImpl> Impl_; -}; + + // Write an emergency message when the memory allocator is corrupted. + // The TThreadedLogBackend object can't be used after this method is called. + void WriteEmergencyData(const TLogRecord& rec); + +private: + class TImpl; + THolder<TImpl> Impl_; +}; class TOwningThreadedLogBackend: private THolder<TLogBackend>, public TThreadedLogBackend { public: diff --git a/library/cpp/logger/ya.make b/library/cpp/logger/ya.make index 00a5263cba..af93cf7412 100644 --- a/library/cpp/logger/ya.make +++ b/library/cpp/logger/ya.make @@ -4,7 +4,7 @@ OWNER( g:util g:base ) - + LIBRARY() GENERATE_ENUM_SERIALIZATION(priority.h) @@ -13,7 +13,7 @@ PEERDIR ( library/cpp/json ) -SRCS( +SRCS( all.h backend.cpp backend_creator.cpp @@ -24,14 +24,14 @@ SRCS( GLOBAL file_creator.cpp filter.cpp filter_creator.cpp - log.cpp + log.cpp null.cpp GLOBAL null_creator.cpp priority.h record.h rotating_file.cpp GLOBAL rotating_file_creator.cpp - stream.cpp + stream.cpp GLOBAL stream_creator.cpp sync_page_cache_file.cpp GLOBAL sync_page_cache_file_creator.cpp @@ -40,8 +40,8 @@ SRCS( thread.cpp thread_creator.cpp GLOBAL uninitialized_creator.cpp -) - -END() +) + +END() RECURSE_FOR_TESTS(ut) diff --git a/library/cpp/lwtrace/control.h b/library/cpp/lwtrace/control.h index 16b24eafd2..4265c622af 100644 --- a/library/cpp/lwtrace/control.h +++ b/library/cpp/lwtrace/control.h @@ -25,12 +25,12 @@ namespace NLWTrace { class IBox: public virtual TThrRefBase { private: bool Owns; - + public: explicit IBox(bool ownsProbe = false) : Owns(ownsProbe) - { - } + { + } bool OwnsProbe() { return Owns; @@ -45,13 +45,13 @@ namespace NLWTrace { class TStaticBox: public IBox { private: TProbe* Probe; - + public: explicit TStaticBox(TProbe* probe) : IBox(false) , Probe(probe) - { - } + { + } TProbe* GetProbe() override { return Probe; diff --git a/library/cpp/lwtrace/example5/lwtrace_example5.cpp b/library/cpp/lwtrace/example5/lwtrace_example5.cpp index 1c324473c2..563f21890f 100644 --- a/library/cpp/lwtrace/example5/lwtrace_example5.cpp +++ b/library/cpp/lwtrace/example5/lwtrace_example5.cpp @@ -3,7 +3,7 @@ template <ui64 N> ui64 Fact() { - ui64 result = N * Fact<N - 1>(); + ui64 result = N * Fact<N - 1>(); #ifndef LWTRACE_DISABLE // Note that probe is create on the first pass @@ -23,8 +23,8 @@ ui64 Fact<0>() { } int main() { - Fact<6>(); // First run is required to create probes we can use later in trace query + Fact<6>(); // First run is required to create probes we can use later in trace query NLWTrace::StartLwtraceFromEnv(); // parse trace query and create trace session - Cout << Fact<6>() << Endl; // actually trigger probes + Cout << Fact<6>() << Endl; // actually trigger probes return 0; } diff --git a/library/cpp/lwtrace/mon/analytics/all.h b/library/cpp/lwtrace/mon/analytics/all.h index 02ddfb83f2..52616bd184 100644 --- a/library/cpp/lwtrace/mon/analytics/all.h +++ b/library/cpp/lwtrace/mon/analytics/all.h @@ -1,8 +1,8 @@ #pragma once -#include "csv_output.h" -#include "data.h" -#include "html_output.h" -#include "json_output.h" -#include "transform.h" -#include "util.h" +#include "csv_output.h" +#include "data.h" +#include "html_output.h" +#include "json_output.h" +#include "transform.h" +#include "util.h" diff --git a/library/cpp/lwtrace/mon/analytics/csv_output.h b/library/cpp/lwtrace/mon/analytics/csv_output.h index 90ded32f5d..fa79c91e31 100644 --- a/library/cpp/lwtrace/mon/analytics/csv_output.h +++ b/library/cpp/lwtrace/mon/analytics/csv_output.h @@ -3,7 +3,7 @@ #include <util/string/printf.h> #include <util/stream/str.h> #include <util/generic/set.h> -#include "data.h" +#include "data.h" namespace NAnalytics { diff --git a/library/cpp/lwtrace/mon/analytics/html_output.h b/library/cpp/lwtrace/mon/analytics/html_output.h index f775f216b9..96c6215544 100644 --- a/library/cpp/lwtrace/mon/analytics/html_output.h +++ b/library/cpp/lwtrace/mon/analytics/html_output.h @@ -3,7 +3,7 @@ #include <util/string/printf.h> #include <util/stream/str.h> #include <util/generic/set.h> -#include "data.h" +#include "data.h" namespace NAnalytics { diff --git a/library/cpp/lwtrace/mon/analytics/json_output.h b/library/cpp/lwtrace/mon/analytics/json_output.h index 189f9802d3..51ce8da3bd 100644 --- a/library/cpp/lwtrace/mon/analytics/json_output.h +++ b/library/cpp/lwtrace/mon/analytics/json_output.h @@ -5,8 +5,8 @@ #include <util/string/vector.h> #include <util/generic/set.h> #include <util/generic/hash_set.h> -#include "data.h" -#include "util.h" +#include "data.h" +#include "util.h" namespace NAnalytics { diff --git a/library/cpp/lwtrace/mon/analytics/transform.h b/library/cpp/lwtrace/mon/analytics/transform.h index f7dc9adb5b..0ba4a27dcd 100644 --- a/library/cpp/lwtrace/mon/analytics/transform.h +++ b/library/cpp/lwtrace/mon/analytics/transform.h @@ -1,6 +1,6 @@ #pragma once -#include "data.h" +#include "data.h" namespace NAnalytics { diff --git a/library/cpp/lwtrace/mon/analytics/util.h b/library/cpp/lwtrace/mon/analytics/util.h index e07d06cc43..1864a3524e 100644 --- a/library/cpp/lwtrace/mon/analytics/util.h +++ b/library/cpp/lwtrace/mon/analytics/util.h @@ -1,6 +1,6 @@ #pragma once -#include "data.h" +#include "data.h" #include <util/generic/algorithm.h> #include <util/generic/hash_set.h> #include <util/string/vector.h> @@ -60,9 +60,9 @@ inline double MinValue(const TString& nameAndOpts, const TTable& table) if (stack) { return 0.0; } else { - auto zero = 0.0; - - return AccumulateIfExist(name, table, 1.0 / zero /*+inf*/, [] (double x, double y) { + auto zero = 0.0; + + return AccumulateIfExist(name, table, 1.0 / zero /*+inf*/, [] (double x, double y) { return Min(x, y); }); } @@ -79,9 +79,9 @@ inline double MaxValue(const TString& nameAndOpts, const TTable& table) return x + y; }); } else { - auto zero = 0.0; - - return AccumulateIfExist(name, table, -1.0 / zero /*-inf*/, [] (double x, double y) { + auto zero = 0.0; + + return AccumulateIfExist(name, table, -1.0 / zero /*-inf*/, [] (double x, double y) { return Max(x, y); }); } diff --git a/library/cpp/lwtrace/rwspinlock.h b/library/cpp/lwtrace/rwspinlock.h index 7c518ec49e..61eec00da5 100644 --- a/library/cpp/lwtrace/rwspinlock.h +++ b/library/cpp/lwtrace/rwspinlock.h @@ -54,7 +54,7 @@ struct TRWSpinLock { SpinLockPause(); } - while (!AtomicCas(&State, TAtomicBase(-1), 1)) { + while (!AtomicCas(&State, TAtomicBase(-1), 1)) { SpinLockPause(); } } diff --git a/library/cpp/malloc/api/helpers/io.cpp b/library/cpp/malloc/api/helpers/io.cpp index 5177969f4d..3f97121298 100644 --- a/library/cpp/malloc/api/helpers/io.cpp +++ b/library/cpp/malloc/api/helpers/io.cpp @@ -1,10 +1,10 @@ #include <library/cpp/malloc/api/malloc.h> - + #include <util/stream/output.h> - -using namespace NMalloc; - -template <> + +using namespace NMalloc; + +template <> void Out<TMallocInfo>(IOutputStream& out, const TMallocInfo& info) { - out << "malloc (name = " << info.Name << ")"; -} + out << "malloc (name = " << info.Name << ")"; +} diff --git a/library/cpp/malloc/api/helpers/ya.make b/library/cpp/malloc/api/helpers/ya.make index 62875bca0e..132da29b00 100644 --- a/library/cpp/malloc/api/helpers/ya.make +++ b/library/cpp/malloc/api/helpers/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - +LIBRARY() + OWNER(pg) -PEERDIR( +PEERDIR( library/cpp/malloc/api -) - -SRCS( - io.cpp -) - -END() +) + +SRCS( + io.cpp +) + +END() diff --git a/library/cpp/malloc/api/malloc.cpp b/library/cpp/malloc/api/malloc.cpp index eed1c58a38..bcfe2b66d8 100644 --- a/library/cpp/malloc/api/malloc.cpp +++ b/library/cpp/malloc/api/malloc.cpp @@ -1,13 +1,13 @@ -#include <stdlib.h> +#include <stdlib.h> #include <stdio.h> #include "malloc.h" -namespace { +namespace { bool SetEmptyParam(const char*, const char*) { - return false; - } - + return false; + } + const char* GetEmptyParam(const char*) { return nullptr; } @@ -15,18 +15,18 @@ namespace { bool CheckEmptyParam(const char*, bool defaultValue) { return defaultValue; } -} +} -namespace NMalloc { - volatile bool IsAllocatorCorrupted = false; - - TMallocInfo::TMallocInfo() - : Name() - , SetParam(SetEmptyParam) +namespace NMalloc { + volatile bool IsAllocatorCorrupted = false; + + TMallocInfo::TMallocInfo() + : Name() + , SetParam(SetEmptyParam) , GetParam(GetEmptyParam) , CheckParam(CheckEmptyParam) - { - } + { + } void AbortFromCorruptedAllocator(const char* errorMessage) { errorMessage = errorMessage ? errorMessage : "<unspecified>"; diff --git a/library/cpp/malloc/api/malloc.h b/library/cpp/malloc/api/malloc.h index ebd545d6dd..9df6f0c7cf 100644 --- a/library/cpp/malloc/api/malloc.h +++ b/library/cpp/malloc/api/malloc.h @@ -6,10 +6,10 @@ namespace NMalloc { struct TMallocInfo { TMallocInfo(); - + const char* Name; - bool (*SetParam)(const char* param, const char* value); + bool (*SetParam)(const char* param, const char* value); const char* (*GetParam)(const char* param); bool (*CheckParam)(const char* param, bool defaultValue); diff --git a/library/cpp/malloc/api/ut/ya.make b/library/cpp/malloc/api/ut/ya.make index e57225b45d..99ea9e5fd6 100644 --- a/library/cpp/malloc/api/ut/ya.make +++ b/library/cpp/malloc/api/ut/ya.make @@ -2,10 +2,10 @@ UNITTEST() OWNER(nga) -PEERDIR( +PEERDIR( library/cpp/malloc/api/helpers -) - +) + SRCS( ut.cpp ) diff --git a/library/cpp/malloc/jemalloc/malloc-info.cpp b/library/cpp/malloc/jemalloc/malloc-info.cpp index 2643ca4766..df2df745a4 100644 --- a/library/cpp/malloc/jemalloc/malloc-info.cpp +++ b/library/cpp/malloc/jemalloc/malloc-info.cpp @@ -1,65 +1,65 @@ #include <library/cpp/malloc/api/malloc.h> -using namespace NMalloc; - -#if defined(_MSC_VER) -TMallocInfo NMalloc::MallocInfo() { - TMallocInfo r; - r.Name = "jemalloc"; - return r; -} -#else -#include <strings.h> -#include <stdlib.h> -#include <inttypes.h> - -#include <contrib/libs/jemalloc/include/jemalloc/jemalloc.h> - -namespace { +using namespace NMalloc; + +#if defined(_MSC_VER) +TMallocInfo NMalloc::MallocInfo() { + TMallocInfo r; + r.Name = "jemalloc"; + return r; +} +#else +#include <strings.h> +#include <stdlib.h> +#include <inttypes.h> + +#include <contrib/libs/jemalloc/include/jemalloc/jemalloc.h> + +namespace { bool JESetParam(const char* param, const char*) { - if (param) { - if (strcmp(param, "j:reset_epoch") == 0) { - uint64_t epoch = 1; - size_t sz = sizeof(epoch); - - mallctl("epoch", &epoch, &sz, &epoch, sz); - - return true; - } - - return false; - } - - return false; - } - - const char* JEGetParam(const char* param) { - if (param) { - if (strcmp(param, "allocated") == 0) { - JESetParam("j:reset_epoch", nullptr); - - size_t allocated = 0; - size_t sz = sizeof(allocated); - - mallctl("stats.allocated", &allocated, &sz, nullptr, 0); - - static_assert(sizeof(size_t) == sizeof(void*), "fix me"); - - return (const char*)(void*)allocated; - } - - return nullptr; - } - - return nullptr; - } -} - + if (param) { + if (strcmp(param, "j:reset_epoch") == 0) { + uint64_t epoch = 1; + size_t sz = sizeof(epoch); + + mallctl("epoch", &epoch, &sz, &epoch, sz); + + return true; + } + + return false; + } + + return false; + } + + const char* JEGetParam(const char* param) { + if (param) { + if (strcmp(param, "allocated") == 0) { + JESetParam("j:reset_epoch", nullptr); + + size_t allocated = 0; + size_t sz = sizeof(allocated); + + mallctl("stats.allocated", &allocated, &sz, nullptr, 0); + + static_assert(sizeof(size_t) == sizeof(void*), "fix me"); + + return (const char*)(void*)allocated; + } + + return nullptr; + } + + return nullptr; + } +} + TMallocInfo NMalloc::MallocInfo() { TMallocInfo r; r.Name = "jemalloc"; - r.SetParam = JESetParam; - r.GetParam = JEGetParam; + r.SetParam = JESetParam; + r.GetParam = JEGetParam; return r; } -#endif +#endif diff --git a/library/cpp/malloc/jemalloc/ya.make b/library/cpp/malloc/jemalloc/ya.make index 99db474eab..672f72e6d9 100644 --- a/library/cpp/malloc/jemalloc/ya.make +++ b/library/cpp/malloc/jemalloc/ya.make @@ -4,18 +4,18 @@ NO_UTIL() OWNER(nga) -IF (OS_ANDROID) - PEERDIR( +IF (OS_ANDROID) + PEERDIR( library/cpp/malloc/system - ) -ELSE() - PEERDIR( + ) +ELSE() + PEERDIR( library/cpp/malloc/api - contrib/libs/jemalloc - ) - SRCS( - malloc-info.cpp - ) -ENDIF() + contrib/libs/jemalloc + ) + SRCS( + malloc-info.cpp + ) +ENDIF() END() diff --git a/library/cpp/malloc/ya.make b/library/cpp/malloc/ya.make index 0ec9db71d2..b968204ad9 100644 --- a/library/cpp/malloc/ya.make +++ b/library/cpp/malloc/ya.make @@ -9,18 +9,18 @@ RECURSE( nalf sample-client system - mimalloc - mimalloc/link_test - hu - hu/link_test + mimalloc + mimalloc/link_test + hu + hu/link_test ) - + IF (NOT OS_WINDOWS) - RECURSE( + RECURSE( calloc calloc/tests calloc/calloc_profile_diff calloc/calloc_profile_scan calloc/calloc_profile_scan/ut - ) + ) ENDIF() diff --git a/library/cpp/messagebus/acceptor.cpp b/library/cpp/messagebus/acceptor.cpp index 64a38619c2..3136ff3284 100644 --- a/library/cpp/messagebus/acceptor.cpp +++ b/library/cpp/messagebus/acceptor.cpp @@ -100,7 +100,7 @@ void TAcceptor::Act(TDefaultTag) { ++Stats.AcceptSuccessCount; } - skipAccept:; + skipAccept:; } Channel->EnableRead(); diff --git a/library/cpp/messagebus/acceptor.h b/library/cpp/messagebus/acceptor.h index 57cb010bf2..6f645cdfa1 100644 --- a/library/cpp/messagebus/acceptor.h +++ b/library/cpp/messagebus/acceptor.h @@ -11,50 +11,50 @@ #include <util/system/event.h> -namespace NBus { - namespace NPrivate { - class TAcceptor - : public NEventLoop::IEventHandler, - private ::NActor::TActor<TAcceptor> { - friend struct TBusSessionImpl; - friend class ::NActor::TActor<TAcceptor>; +namespace NBus { + namespace NPrivate { + class TAcceptor + : public NEventLoop::IEventHandler, + private ::NActor::TActor<TAcceptor> { + friend struct TBusSessionImpl; + friend class ::NActor::TActor<TAcceptor>; - public: - TAcceptor(TBusSessionImpl* session, ui64 acceptorId, SOCKET socket, const TNetAddr& addr); + public: + TAcceptor(TBusSessionImpl* session, ui64 acceptorId, SOCKET socket, const TNetAddr& addr); - void HandleEvent(SOCKET socket, void* cookie) override; + void HandleEvent(SOCKET socket, void* cookie) override; - void Shutdown(); + void Shutdown(); - inline ::NActor::TActor<TAcceptor>* GetActor() { - return this; - } + inline ::NActor::TActor<TAcceptor>* GetActor() { + return this; + } - private: - void SendStatus(TInstant now); - void Act(::NActor::TDefaultTag); + private: + void SendStatus(TInstant now); + void Act(::NActor::TDefaultTag); - private: - const ui64 AcceptorId; + private: + const ui64 AcceptorId; - TBusSessionImpl* const Session; - NEventLoop::TChannelPtr Channel; + TBusSessionImpl* const Session; + NEventLoop::TChannelPtr Channel; - TAcceptorStatus Stats; + TAcceptorStatus Stats; - TAtomicShutdownState ShutdownState; + TAtomicShutdownState ShutdownState; - struct TGranStatus { - TGranStatus(TDuration gran) - : Listen(gran) - { - } + struct TGranStatus { + TGranStatus(TDuration gran) + : Listen(gran) + { + } - TGranUp<TAcceptorStatus> Listen; - }; + TGranUp<TAcceptorStatus> Listen; + }; - TGranStatus GranStatus; + TGranStatus GranStatus; }; - } -} + } +} diff --git a/library/cpp/messagebus/acceptor_status.cpp b/library/cpp/messagebus/acceptor_status.cpp index 5006ff68ae..fcc982e752 100644 --- a/library/cpp/messagebus/acceptor_status.cpp +++ b/library/cpp/messagebus/acceptor_status.cpp @@ -53,13 +53,13 @@ TString TAcceptorStatus::PrintToString() const { if (AcceptErrorCount > 0) { p.AddRow("last accept error", - TString() + LastSystemErrorText(LastAcceptErrorErrno) + " at " + LastAcceptErrorInstant.ToString()); + TString() + LastSystemErrorText(LastAcceptErrorErrno) + " at " + LastAcceptErrorInstant.ToString()); } p.AddRow("accept success count", LeftPad(AcceptSuccessCount, 4)); if (AcceptSuccessCount > 0) { p.AddRow("last accept success", - TString() + "at " + LastAcceptSuccessInstant.ToString()); + TString() + "at " + LastAcceptSuccessInstant.ToString()); } ss << p.PrintToString(); diff --git a/library/cpp/messagebus/acceptor_status.h b/library/cpp/messagebus/acceptor_status.h index 6aa1404f4d..81b5b26485 100644 --- a/library/cpp/messagebus/acceptor_status.h +++ b/library/cpp/messagebus/acceptor_status.h @@ -4,32 +4,32 @@ #include <util/network/init.h> -namespace NBus { - namespace NPrivate { - struct TAcceptorStatus { - bool Summary; +namespace NBus { + namespace NPrivate { + struct TAcceptorStatus { + bool Summary; - ui64 AcceptorId; + ui64 AcceptorId; - SOCKET Fd; + SOCKET Fd; - TNetAddr ListenAddr; + TNetAddr ListenAddr; - unsigned AcceptSuccessCount; - TInstant LastAcceptSuccessInstant; + unsigned AcceptSuccessCount; + TInstant LastAcceptSuccessInstant; - unsigned AcceptErrorCount; - TInstant LastAcceptErrorInstant; - int LastAcceptErrorErrno; + unsigned AcceptErrorCount; + TInstant LastAcceptErrorInstant; + int LastAcceptErrorErrno; - void ResetIncremental(); + void ResetIncremental(); - TAcceptorStatus(); + TAcceptorStatus(); - TAcceptorStatus& operator+=(const TAcceptorStatus& that); + TAcceptorStatus& operator+=(const TAcceptorStatus& that); - TString PrintToString() const; - }; + TString PrintToString() const; + }; - } -} + } +} diff --git a/library/cpp/messagebus/actor/actor.h b/library/cpp/messagebus/actor/actor.h index 9b8f20298a..1f5af55097 100644 --- a/library/cpp/messagebus/actor/actor.h +++ b/library/cpp/messagebus/actor/actor.h @@ -7,58 +7,58 @@ #include <util/system/yassert.h> namespace NActor { - class IActor: protected IWorkItem { - public: - // TODO: make private - TTasks Tasks; - - public: - virtual void ScheduleHereV() = 0; - virtual void ScheduleV() = 0; - virtual void ScheduleHereAtMostOnceV() = 0; - - // TODO: make private - virtual void RefV() = 0; - virtual void UnRefV() = 0; - - // mute warnings - ~IActor() override { - } - }; - - struct TDefaultTag {}; - - template <typename TThis, typename TTag = TDefaultTag> - class TActor: public IActor { - private: - TExecutor* const Executor; - - public: - TActor(TExecutor* executor) - : Executor(executor) - { - } - - void AddTaskFromActorLoop() { - bool schedule = Tasks.AddTask(); - // TODO: check thread id - Y_ASSERT(!schedule); - } - - /** + class IActor: protected IWorkItem { + public: + // TODO: make private + TTasks Tasks; + + public: + virtual void ScheduleHereV() = 0; + virtual void ScheduleV() = 0; + virtual void ScheduleHereAtMostOnceV() = 0; + + // TODO: make private + virtual void RefV() = 0; + virtual void UnRefV() = 0; + + // mute warnings + ~IActor() override { + } + }; + + struct TDefaultTag {}; + + template <typename TThis, typename TTag = TDefaultTag> + class TActor: public IActor { + private: + TExecutor* const Executor; + + public: + TActor(TExecutor* executor) + : Executor(executor) + { + } + + void AddTaskFromActorLoop() { + bool schedule = Tasks.AddTask(); + // TODO: check thread id + Y_ASSERT(!schedule); + } + + /** * Schedule actor. * * If actor is sleeping, then actor will be executed right now. * If actor is executing right now, it will be executed one more time. * If this method is called multiple time, actor will be re-executed no more than one more time. */ - void Schedule() { - if (Tasks.AddTask()) { - EnqueueWork(); - } + void Schedule() { + if (Tasks.AddTask()) { + EnqueueWork(); + } } - /** + /** * Schedule actor, execute it in current thread. * * If actor is running, continue executing where it is executing. @@ -66,79 +66,79 @@ namespace NActor { * * Operation is useful for tasks that are likely to complete quickly. */ - void ScheduleHere() { - if (Tasks.AddTask()) { - Loop(); - } + void ScheduleHere() { + if (Tasks.AddTask()) { + Loop(); + } } - /** + /** * Schedule actor, execute in current thread no more than once. * * If actor is running, continue executing where it is executing. * If actor is sleeping, execute one iteration here, and if actor got new tasks, * reschedule it in worker pool. */ - void ScheduleHereAtMostOnce() { - if (Tasks.AddTask()) { - bool fetched = Tasks.FetchTask(); - Y_VERIFY(fetched, "happens"); - - DoAct(); - - // if someone added more tasks, schedule them - if (Tasks.FetchTask()) { - bool added = Tasks.AddTask(); - Y_VERIFY(!added, "happens"); - EnqueueWork(); - } - } - } - - void ScheduleHereV() override { - ScheduleHere(); - } - void ScheduleV() override { - Schedule(); - } - void ScheduleHereAtMostOnceV() override { - ScheduleHereAtMostOnce(); - } - void RefV() override { - GetThis()->Ref(); - } - void UnRefV() override { - GetThis()->UnRef(); - } - - private: - TThis* GetThis() { - return static_cast<TThis*>(this); - } - - void EnqueueWork() { - GetThis()->Ref(); - Executor->EnqueueWork({this}); - } - - void DoAct() { - WHAT_THREAD_DOES_PUSH_POP_CURRENT_FUNC(); - - GetThis()->Act(TTag()); - } - - void Loop() { - // TODO: limit number of iterations - while (Tasks.FetchTask()) { - DoAct(); + void ScheduleHereAtMostOnce() { + if (Tasks.AddTask()) { + bool fetched = Tasks.FetchTask(); + Y_VERIFY(fetched, "happens"); + + DoAct(); + + // if someone added more tasks, schedule them + if (Tasks.FetchTask()) { + bool added = Tasks.AddTask(); + Y_VERIFY(!added, "happens"); + EnqueueWork(); + } } } - void DoWork() override { - Y_ASSERT(GetThis()->RefCount() >= 1); - Loop(); - GetThis()->UnRef(); - } - }; - -} + void ScheduleHereV() override { + ScheduleHere(); + } + void ScheduleV() override { + Schedule(); + } + void ScheduleHereAtMostOnceV() override { + ScheduleHereAtMostOnce(); + } + void RefV() override { + GetThis()->Ref(); + } + void UnRefV() override { + GetThis()->UnRef(); + } + + private: + TThis* GetThis() { + return static_cast<TThis*>(this); + } + + void EnqueueWork() { + GetThis()->Ref(); + Executor->EnqueueWork({this}); + } + + void DoAct() { + WHAT_THREAD_DOES_PUSH_POP_CURRENT_FUNC(); + + GetThis()->Act(TTag()); + } + + void Loop() { + // TODO: limit number of iterations + while (Tasks.FetchTask()) { + DoAct(); + } + } + + void DoWork() override { + Y_ASSERT(GetThis()->RefCount() >= 1); + Loop(); + GetThis()->UnRef(); + } + }; + +} diff --git a/library/cpp/messagebus/actor/actor_ut.cpp b/library/cpp/messagebus/actor/actor_ut.cpp index b76ab55bfa..20dee53e2d 100644 --- a/library/cpp/messagebus/actor/actor_ut.cpp +++ b/library/cpp/messagebus/actor/actor_ut.cpp @@ -11,14 +11,14 @@ using namespace NActor; template <typename TThis> -struct TTestActorBase: public TAtomicRefCount<TThis>, public TActor<TThis> { +struct TTestActorBase: public TAtomicRefCount<TThis>, public TActor<TThis> { TTestSync Started; TTestSync Acted; TTestActorBase(TExecutor* executor) : TActor<TThis>(executor) - { - } + { + } void Act(TDefaultTag) { Started.Inc(); @@ -27,23 +27,23 @@ struct TTestActorBase: public TAtomicRefCount<TThis>, public TActor<TThis> { } }; -struct TNopActor: public TTestActorBase<TNopActor> { +struct TNopActor: public TTestActorBase<TNopActor> { TObjectCounter<TNopActor> AllocCounter; TNopActor(TExecutor* executor) : TTestActorBase<TNopActor>(executor) - { - } + { + } void Act2() { } }; -struct TWaitForSignalActor: public TTestActorBase<TWaitForSignalActor> { +struct TWaitForSignalActor: public TTestActorBase<TWaitForSignalActor> { TWaitForSignalActor(TExecutor* executor) : TTestActorBase<TWaitForSignalActor>(executor) - { - } + { + } TSystemEvent WaitFor; @@ -52,7 +52,7 @@ struct TWaitForSignalActor: public TTestActorBase<TWaitForSignalActor> { } }; -struct TDecrementAndSendActor: public TTestActorBase<TDecrementAndSendActor>, public TQueueInActor<TDecrementAndSendActor, int> { +struct TDecrementAndSendActor: public TTestActorBase<TDecrementAndSendActor>, public TQueueInActor<TDecrementAndSendActor, int> { TSystemEvent Done; TDecrementAndSendActor* Next; @@ -60,8 +60,8 @@ struct TDecrementAndSendActor: public TTestActorBase<TDecrementAndSendActor>, pu TDecrementAndSendActor(TExecutor* executor) : TTestActorBase<TDecrementAndSendActor>(executor) , Next(nullptr) - { - } + { + } void ProcessItem(TDefaultTag, TDefaultTag, int n) { if (n == 0) { @@ -129,7 +129,7 @@ Y_UNIT_TEST_SUITE(TActor) { TExecutor executor(queueSize); - TVector<TIntrusivePtr<TDecrementAndSendActor>> actors; + TVector<TIntrusivePtr<TDecrementAndSendActor>> actors; for (int i = 0; i < actorCount; ++i) { actors.push_back(new TDecrementAndSendActor(&executor)); } diff --git a/library/cpp/messagebus/actor/executor.cpp b/library/cpp/messagebus/actor/executor.cpp index 7a2227a458..4c428754f3 100644 --- a/library/cpp/messagebus/actor/executor.cpp +++ b/library/cpp/messagebus/actor/executor.cpp @@ -20,10 +20,10 @@ namespace { struct TRecord { TAtomic MaxQueueSize; - TRecord() - : MaxQueueSize() - { - } + TRecord() + : MaxQueueSize() + { + } TExecutorHistory::THistoryRecord Capture() { TExecutorHistory::THistoryRecord r; @@ -70,8 +70,8 @@ namespace { } -Y_POD_STATIC_THREAD(TExecutor*) -ThreadCurrentExecutor; +Y_POD_STATIC_THREAD(TExecutor*) +ThreadCurrentExecutor; static const char* NoLocation = "nowhere"; @@ -80,80 +80,80 @@ struct TExecutorWorkerThreadLocalData { }; static TExecutorWorkerThreadLocalData WorkerNoThreadLocalData; -Y_POD_STATIC_THREAD(TExecutorWorkerThreadLocalData) -WorkerThreadLocalData; +Y_POD_STATIC_THREAD(TExecutorWorkerThreadLocalData) +WorkerThreadLocalData; namespace NActor { - struct TExecutorWorker { - TExecutor* const Executor; - TThread Thread; - const char** WhatThreadDoesLocation; - TExecutorWorkerThreadLocalData* ThreadLocalData; - - TExecutorWorker(TExecutor* executor) - : Executor(executor) - , Thread(RunThreadProc, this) - , WhatThreadDoesLocation(&NoLocation) - , ThreadLocalData(&::WorkerNoThreadLocalData) - { - Thread.Start(); - } - - void Run() { - WhatThreadDoesLocation = ::WhatThreadDoesLocation(); - AtomicSet(ThreadLocalData, &::WorkerThreadLocalData); - WHAT_THREAD_DOES_PUSH_POP_CURRENT_FUNC(); - Executor->RunWorker(); - } - - static void* RunThreadProc(void* thiz0) { - TExecutorWorker* thiz = (TExecutorWorker*)thiz0; - thiz->Run(); - return nullptr; - } - }; - - struct TExecutor::TImpl { - TExecutor* const Executor; - THistoryInternal History; + struct TExecutorWorker { + TExecutor* const Executor; + TThread Thread; + const char** WhatThreadDoesLocation; + TExecutorWorkerThreadLocalData* ThreadLocalData; + + TExecutorWorker(TExecutor* executor) + : Executor(executor) + , Thread(RunThreadProc, this) + , WhatThreadDoesLocation(&NoLocation) + , ThreadLocalData(&::WorkerNoThreadLocalData) + { + Thread.Start(); + } + + void Run() { + WhatThreadDoesLocation = ::WhatThreadDoesLocation(); + AtomicSet(ThreadLocalData, &::WorkerThreadLocalData); + WHAT_THREAD_DOES_PUSH_POP_CURRENT_FUNC(); + Executor->RunWorker(); + } + + static void* RunThreadProc(void* thiz0) { + TExecutorWorker* thiz = (TExecutorWorker*)thiz0; + thiz->Run(); + return nullptr; + } + }; + + struct TExecutor::TImpl { + TExecutor* const Executor; + THistoryInternal History; TSystemEvent HelperStopSignal; - TThread HelperThread; + TThread HelperThread; - TImpl(TExecutor* executor) - : Executor(executor) - , HelperThread(HelperThreadProc, this) - { - } + TImpl(TExecutor* executor) + : Executor(executor) + , HelperThread(HelperThreadProc, this) + { + } - void RunHelper() { - ui64 nowSeconds = TInstant::Now().Seconds(); - for (;;) { - TInstant nextStop = TInstant::Seconds(nowSeconds + 1) + TDuration::MilliSeconds(RandomNumber<ui32>(1000)); + void RunHelper() { + ui64 nowSeconds = TInstant::Now().Seconds(); + for (;;) { + TInstant nextStop = TInstant::Seconds(nowSeconds + 1) + TDuration::MilliSeconds(RandomNumber<ui32>(1000)); - if (HelperStopSignal.WaitD(nextStop)) { - return; - } + if (HelperStopSignal.WaitD(nextStop)) { + return; + } - nowSeconds = nextStop.Seconds(); + nowSeconds = nextStop.Seconds(); - THistoryInternal::TRecord& record = History.GetNowRecord(nowSeconds); + THistoryInternal::TRecord& record = History.GetNowRecord(nowSeconds); - ui32 maxQueueSize = Executor->GetMaxQueueSizeAndClear(); - if (maxQueueSize > record.MaxQueueSize) { - AtomicSet(record.MaxQueueSize, maxQueueSize); - } + ui32 maxQueueSize = Executor->GetMaxQueueSizeAndClear(); + if (maxQueueSize > record.MaxQueueSize) { + AtomicSet(record.MaxQueueSize, maxQueueSize); + } } } - static void* HelperThreadProc(void* impl0) { - TImpl* impl = (TImpl*)impl0; - impl->RunHelper(); - return nullptr; - } - }; + static void* HelperThreadProc(void* impl0) { + TImpl* impl = (TImpl*)impl0; + impl->RunHelper(); + return nullptr; + } + }; -} +} static TExecutor::TConfig MakeConfig(unsigned workerCount) { TExecutor::TConfig config; @@ -296,9 +296,9 @@ TAutoPtr<IWorkItem> TExecutor::DequeueWork() { WorkAvailable.Wait(WorkMutex); } } - - auto& wtls = TlsRef(WorkerThreadLocalData); - + + auto& wtls = TlsRef(WorkerThreadLocalData); + if (queueSize > RelaxedLoad(&wtls.MaxQueueSize)) { RelaxedStore<ui32>(&wtls.MaxQueueSize, queueSize); } @@ -334,5 +334,5 @@ void TExecutor::RunWorker() { RunWorkItem(wi); } - ThreadCurrentExecutor = (TExecutor*)nullptr; + ThreadCurrentExecutor = (TExecutor*)nullptr; } diff --git a/library/cpp/messagebus/actor/executor.h b/library/cpp/messagebus/actor/executor.h index 7292d8be53..4b9bcb1da0 100644 --- a/library/cpp/messagebus/actor/executor.h +++ b/library/cpp/messagebus/actor/executor.h @@ -11,95 +11,95 @@ #include <util/thread/lfqueue.h> namespace NActor { - namespace NPrivate { - struct TExecutorHistory { - struct THistoryRecord { - ui32 MaxQueueSize; - }; - TVector<THistoryRecord> HistoryRecords; - ui64 LastHistoryRecordSecond; - - ui64 FirstHistoryRecordSecond() const { - return LastHistoryRecordSecond - HistoryRecords.size() + 1; - } + namespace NPrivate { + struct TExecutorHistory { + struct THistoryRecord { + ui32 MaxQueueSize; + }; + TVector<THistoryRecord> HistoryRecords; + ui64 LastHistoryRecordSecond; + + ui64 FirstHistoryRecordSecond() const { + return LastHistoryRecordSecond - HistoryRecords.size() + 1; + } + }; + + struct TExecutorStatus { + size_t WorkQueueSize = 0; + TExecutorHistory History; + TString Status; }; + } - struct TExecutorStatus { - size_t WorkQueueSize = 0; - TExecutorHistory History; - TString Status; - }; - } - - class IWorkItem { - public: - virtual ~IWorkItem() { + class IWorkItem { + public: + virtual ~IWorkItem() { } - virtual void DoWork(/* must release this */) = 0; + virtual void DoWork(/* must release this */) = 0; }; - struct TExecutorWorker; + struct TExecutorWorker; - class TExecutor: public TAtomicRefCount<TExecutor> { - friend struct TExecutorWorker; + class TExecutor: public TAtomicRefCount<TExecutor> { + friend struct TExecutorWorker; - public: - struct TConfig { - size_t WorkerCount; - const char* Name; + public: + struct TConfig { + size_t WorkerCount; + const char* Name; - TConfig() - : WorkerCount(1) - , Name() - { - } - }; + TConfig() + : WorkerCount(1) + , Name() + { + } + }; - private: - struct TImpl; - THolder<TImpl> Impl; + private: + struct TImpl; + THolder<TImpl> Impl; - const TConfig Config; + const TConfig Config; - TAtomic ExitWorkers; + TAtomic ExitWorkers; - TVector<TAutoPtr<TExecutorWorker>> WorkerThreads; + TVector<TAutoPtr<TExecutorWorker>> WorkerThreads; - TRingBufferWithSpinLock<IWorkItem*> WorkItems; + TRingBufferWithSpinLock<IWorkItem*> WorkItems; - TMutex WorkMutex; - TCondVar WorkAvailable; + TMutex WorkMutex; + TCondVar WorkAvailable; - public: - explicit TExecutor(size_t workerCount); - TExecutor(const TConfig& config); - ~TExecutor(); + public: + explicit TExecutor(size_t workerCount); + TExecutor(const TConfig& config); + ~TExecutor(); - void Stop(); + void Stop(); - void EnqueueWork(TArrayRef<IWorkItem* const> w); + void EnqueueWork(TArrayRef<IWorkItem* const> w); - size_t GetWorkQueueSize() const; - TString GetStatus() const; - TString GetStatusSingleLine() const; - NPrivate::TExecutorStatus GetStatusRecordInternal() const; + size_t GetWorkQueueSize() const; + TString GetStatus() const; + TString GetStatusSingleLine() const; + NPrivate::TExecutorStatus GetStatusRecordInternal() const; - bool IsInExecutorThread() const; + bool IsInExecutorThread() const; - private: - void Init(); + private: + void Init(); - TAutoPtr<IWorkItem> DequeueWork(); + TAutoPtr<IWorkItem> DequeueWork(); - void ProcessWorkQueueHere(); + void ProcessWorkQueueHere(); - inline void RunWorkItem(TAutoPtr<IWorkItem>); + inline void RunWorkItem(TAutoPtr<IWorkItem>); - void RunWorker(); + void RunWorker(); - ui32 GetMaxQueueSizeAndClear() const; - }; + ui32 GetMaxQueueSizeAndClear() const; + }; - using TExecutorPtr = TIntrusivePtr<TExecutor>; + using TExecutorPtr = TIntrusivePtr<TExecutor>; -} +} diff --git a/library/cpp/messagebus/actor/queue_for_actor.h b/library/cpp/messagebus/actor/queue_for_actor.h index 40fa536b82..f59f229b79 100644 --- a/library/cpp/messagebus/actor/queue_for_actor.h +++ b/library/cpp/messagebus/actor/queue_for_actor.h @@ -9,66 +9,66 @@ #include "temp_tls_vector.h" namespace NActor { - namespace NPrivate { - struct TTagForTl {}; + namespace NPrivate { + struct TTagForTl {}; - } + } - template <typename T> - class TQueueForActor { - private: - TLockFreeStack<T> Queue; + template <typename T> + class TQueueForActor { + private: + TLockFreeStack<T> Queue; - public: - ~TQueueForActor() { - Y_VERIFY(Queue.IsEmpty()); - } + public: + ~TQueueForActor() { + Y_VERIFY(Queue.IsEmpty()); + } - bool IsEmpty() { - return Queue.IsEmpty(); - } + bool IsEmpty() { + return Queue.IsEmpty(); + } - void Enqueue(const T& value) { - Queue.Enqueue(value); - } + void Enqueue(const T& value) { + Queue.Enqueue(value); + } - template <typename TCollection> - void EnqueueAll(const TCollection& all) { - Queue.EnqueueAll(all); - } + template <typename TCollection> + void EnqueueAll(const TCollection& all) { + Queue.EnqueueAll(all); + } - void Clear() { - TVector<T> tmp; - Queue.DequeueAll(&tmp); - } + void Clear() { + TVector<T> tmp; + Queue.DequeueAll(&tmp); + } - template <typename TFunc> - void DequeueAll(const TFunc& func - // TODO: , std::enable_if_t<TFunctionParamCount<TFunc>::Value == 1>* = 0 - ) { - TTempTlsVector<T> temp; + template <typename TFunc> + void DequeueAll(const TFunc& func + // TODO: , std::enable_if_t<TFunctionParamCount<TFunc>::Value == 1>* = 0 + ) { + TTempTlsVector<T> temp; - Queue.DequeueAllSingleConsumer(temp.GetVector()); + Queue.DequeueAllSingleConsumer(temp.GetVector()); - for (typename TVector<T>::reverse_iterator i = temp.GetVector()->rbegin(); i != temp.GetVector()->rend(); ++i) { - func(*i); - } + for (typename TVector<T>::reverse_iterator i = temp.GetVector()->rbegin(); i != temp.GetVector()->rend(); ++i) { + func(*i); + } - temp.Clear(); + temp.Clear(); - if (temp.Capacity() * sizeof(T) > 64 * 1024) { - temp.Shrink(); - } + if (temp.Capacity() * sizeof(T) > 64 * 1024) { + temp.Shrink(); + } } - template <typename TFunc> - void DequeueAllLikelyEmpty(const TFunc& func) { - if (Y_LIKELY(IsEmpty())) { - return; - } + template <typename TFunc> + void DequeueAllLikelyEmpty(const TFunc& func) { + if (Y_LIKELY(IsEmpty())) { + return; + } - DequeueAll(func); + DequeueAll(func); } - }; + }; -} +} diff --git a/library/cpp/messagebus/actor/queue_in_actor.h b/library/cpp/messagebus/actor/queue_in_actor.h index 9865996532..65110967aa 100644 --- a/library/cpp/messagebus/actor/queue_in_actor.h +++ b/library/cpp/messagebus/actor/queue_in_actor.h @@ -6,75 +6,75 @@ #include <functional> namespace NActor { - template <typename TItem> - class IQueueInActor { - public: - virtual void EnqueueAndScheduleV(const TItem& item) = 0; - virtual void DequeueAllV() = 0; - virtual void DequeueAllLikelyEmptyV() = 0; - - virtual ~IQueueInActor() { - } - }; - - template <typename TThis, typename TItem, typename TActorTag = TDefaultTag, typename TQueueTag = TDefaultTag> - class TQueueInActor: public IQueueInActor<TItem> { - typedef TQueueInActor<TThis, TItem, TActorTag, TQueueTag> TSelf; - - public: - // TODO: make protected - TQueueForActor<TItem> QueueInActor; - - private: - TActor<TThis, TActorTag>* GetActor() { - return GetThis(); - } - - TThis* GetThis() { - return static_cast<TThis*>(this); - } - - void ProcessItem(const TItem& item) { - GetThis()->ProcessItem(TActorTag(), TQueueTag(), item); - } - - public: - void EnqueueAndNoSchedule(const TItem& item) { - QueueInActor.Enqueue(item); - } - - void EnqueueAndSchedule(const TItem& item) { - EnqueueAndNoSchedule(item); - GetActor()->Schedule(); - } - - void EnqueueAndScheduleV(const TItem& item) override { - EnqueueAndSchedule(item); - } - - void Clear() { - QueueInActor.Clear(); - } - - void DequeueAll() { - QueueInActor.DequeueAll(std::bind(&TSelf::ProcessItem, this, std::placeholders::_1)); - } - - void DequeueAllV() override { - return DequeueAll(); - } - - void DequeueAllLikelyEmpty() { - QueueInActor.DequeueAllLikelyEmpty(std::bind(&TSelf::ProcessItem, this, std::placeholders::_1)); - } - - void DequeueAllLikelyEmptyV() override { - return DequeueAllLikelyEmpty(); - } - - bool IsEmpty() { - return QueueInActor.IsEmpty(); - } - }; - -} + template <typename TItem> + class IQueueInActor { + public: + virtual void EnqueueAndScheduleV(const TItem& item) = 0; + virtual void DequeueAllV() = 0; + virtual void DequeueAllLikelyEmptyV() = 0; + + virtual ~IQueueInActor() { + } + }; + + template <typename TThis, typename TItem, typename TActorTag = TDefaultTag, typename TQueueTag = TDefaultTag> + class TQueueInActor: public IQueueInActor<TItem> { + typedef TQueueInActor<TThis, TItem, TActorTag, TQueueTag> TSelf; + + public: + // TODO: make protected + TQueueForActor<TItem> QueueInActor; + + private: + TActor<TThis, TActorTag>* GetActor() { + return GetThis(); + } + + TThis* GetThis() { + return static_cast<TThis*>(this); + } + + void ProcessItem(const TItem& item) { + GetThis()->ProcessItem(TActorTag(), TQueueTag(), item); + } + + public: + void EnqueueAndNoSchedule(const TItem& item) { + QueueInActor.Enqueue(item); + } + + void EnqueueAndSchedule(const TItem& item) { + EnqueueAndNoSchedule(item); + GetActor()->Schedule(); + } + + void EnqueueAndScheduleV(const TItem& item) override { + EnqueueAndSchedule(item); + } + + void Clear() { + QueueInActor.Clear(); + } + + void DequeueAll() { + QueueInActor.DequeueAll(std::bind(&TSelf::ProcessItem, this, std::placeholders::_1)); + } + + void DequeueAllV() override { + return DequeueAll(); + } + + void DequeueAllLikelyEmpty() { + QueueInActor.DequeueAllLikelyEmpty(std::bind(&TSelf::ProcessItem, this, std::placeholders::_1)); + } + + void DequeueAllLikelyEmptyV() override { + return DequeueAllLikelyEmpty(); + } + + bool IsEmpty() { + return QueueInActor.IsEmpty(); + } + }; + +} diff --git a/library/cpp/messagebus/actor/ring_buffer.h b/library/cpp/messagebus/actor/ring_buffer.h index ec5706f7c7..8ca76bdb91 100644 --- a/library/cpp/messagebus/actor/ring_buffer.h +++ b/library/cpp/messagebus/actor/ring_buffer.h @@ -95,11 +95,11 @@ public: } for (size_t i = 0; i < firstSize; ++i) { - Data[WritePos + i] = value[i]; + Data[WritePos + i] = value[i]; } for (size_t i = 0; i < secondSize; ++i) { - Data[i] = value[firstSize + i]; + Data[i] = value[firstSize + i]; } WritePos = (WritePos + value.size()) & CapacityMask; diff --git a/library/cpp/messagebus/actor/ring_buffer_ut.cpp b/library/cpp/messagebus/actor/ring_buffer_ut.cpp index bdb379b3a9..27926b5f2c 100644 --- a/library/cpp/messagebus/actor/ring_buffer_ut.cpp +++ b/library/cpp/messagebus/actor/ring_buffer_ut.cpp @@ -11,11 +11,11 @@ Y_UNIT_TEST_SUITE(RingBuffer) { unsigned NextPush; unsigned NextPop; - TRingBufferTester() - : NextPush() - , NextPop() - { - } + TRingBufferTester() + : NextPush() + , NextPop() + { + } void Push() { //Cerr << "push " << NextPush << "\n"; diff --git a/library/cpp/messagebus/actor/ring_buffer_with_spin_lock.h b/library/cpp/messagebus/actor/ring_buffer_with_spin_lock.h index f0b7cd90e4..39e1d05506 100644 --- a/library/cpp/messagebus/actor/ring_buffer_with_spin_lock.h +++ b/library/cpp/messagebus/actor/ring_buffer_with_spin_lock.h @@ -14,8 +14,8 @@ private: public: TRingBufferWithSpinLock() : CachedSize(0) - { - } + { + } void Push(const T& t) { PushAll(t); diff --git a/library/cpp/messagebus/actor/tasks.h b/library/cpp/messagebus/actor/tasks.h index 31d35931d2..e3b3f0f504 100644 --- a/library/cpp/messagebus/actor/tasks.h +++ b/library/cpp/messagebus/actor/tasks.h @@ -4,45 +4,45 @@ #include <util/system/yassert.h> namespace NActor { - class TTasks { - enum { - // order of values is important - E_WAITING, - E_RUNNING_NO_TASKS, - E_RUNNING_GOT_TASKS, - }; - - private: - TAtomic State; - - public: - TTasks() - : State(E_WAITING) - { + class TTasks { + enum { + // order of values is important + E_WAITING, + E_RUNNING_NO_TASKS, + E_RUNNING_GOT_TASKS, + }; + + private: + TAtomic State; + + public: + TTasks() + : State(E_WAITING) + { } - // @return true iff caller have to either schedule task or execute it - bool AddTask() { - // High contention case optimization: AtomicGet is cheaper than AtomicSwap. - if (E_RUNNING_GOT_TASKS == AtomicGet(State)) { - return false; - } + // @return true iff caller have to either schedule task or execute it + bool AddTask() { + // High contention case optimization: AtomicGet is cheaper than AtomicSwap. + if (E_RUNNING_GOT_TASKS == AtomicGet(State)) { + return false; + } - TAtomicBase oldState = AtomicSwap(&State, E_RUNNING_GOT_TASKS); - return oldState == E_WAITING; + TAtomicBase oldState = AtomicSwap(&State, E_RUNNING_GOT_TASKS); + return oldState == E_WAITING; } - // called by executor - // @return true iff we have to recheck queues - bool FetchTask() { - TAtomicBase newState = AtomicDecrement(State); - if (newState == E_RUNNING_NO_TASKS) { - return true; - } else if (newState == E_WAITING) { - return false; - } - Y_FAIL("unknown"); - } - }; - -} + // called by executor + // @return true iff we have to recheck queues + bool FetchTask() { + TAtomicBase newState = AtomicDecrement(State); + if (newState == E_RUNNING_NO_TASKS) { + return true; + } else if (newState == E_WAITING) { + return false; + } + Y_FAIL("unknown"); + } + }; + +} diff --git a/library/cpp/messagebus/actor/temp_tls_vector.h b/library/cpp/messagebus/actor/temp_tls_vector.h index 675d92f5b0..5b94b2126f 100644 --- a/library/cpp/messagebus/actor/temp_tls_vector.h +++ b/library/cpp/messagebus/actor/temp_tls_vector.h @@ -5,13 +5,13 @@ #include <util/generic/vector.h> #include <util/system/yassert.h> -template <typename T, typename TTag = void, template <typename, class> class TVectorType = TVector> +template <typename T, typename TTag = void, template <typename, class> class TVectorType = TVector> class TTempTlsVector { private: struct TTagForTls {}; TVectorType<T, std::allocator<T>>* Vector; - + public: TVectorType<T, std::allocator<T>>* GetVector() { return Vector; diff --git a/library/cpp/messagebus/actor/thread_extra.h b/library/cpp/messagebus/actor/thread_extra.h index b5aa151618..efd150a0b2 100644 --- a/library/cpp/messagebus/actor/thread_extra.h +++ b/library/cpp/messagebus/actor/thread_extra.h @@ -3,39 +3,39 @@ #include <util/thread/singleton.h> namespace NTSAN { - template <typename T> - inline void RelaxedStore(volatile T* a, T x) { - static_assert(std::is_integral<T>::value || std::is_pointer<T>::value, "expect std::is_integral<T>::value || std::is_pointer<T>::value"); + template <typename T> + inline void RelaxedStore(volatile T* a, T x) { + static_assert(std::is_integral<T>::value || std::is_pointer<T>::value, "expect std::is_integral<T>::value || std::is_pointer<T>::value"); #ifdef _win_ - *a = x; + *a = x; #else - __atomic_store_n(a, x, __ATOMIC_RELAXED); + __atomic_store_n(a, x, __ATOMIC_RELAXED); #endif - } + } - template <typename T> - inline T RelaxedLoad(volatile T* a) { + template <typename T> + inline T RelaxedLoad(volatile T* a) { #ifdef _win_ - return *a; + return *a; #else - return __atomic_load_n(a, __ATOMIC_RELAXED); + return __atomic_load_n(a, __ATOMIC_RELAXED); #endif - } + } } void SetCurrentThreadName(const char* name); -namespace NThreadExtra { - namespace NPrivate { - template <typename TValue, typename TTag> - struct TValueHolder { - TValue Value; - }; - } -} +namespace NThreadExtra { + namespace NPrivate { + template <typename TValue, typename TTag> + struct TValueHolder { + TValue Value; + }; + } +} template <typename TValue, typename TTag> static inline TValue* FastTlsSingletonWithTag() { - return &FastTlsSingleton< ::NThreadExtra::NPrivate::TValueHolder<TValue, TTag>>()->Value; + return &FastTlsSingleton< ::NThreadExtra::NPrivate::TValueHolder<TValue, TTag>>()->Value; } diff --git a/library/cpp/messagebus/actor/what_thread_does.cpp b/library/cpp/messagebus/actor/what_thread_does.cpp index bebb6a888c..5960093a6b 100644 --- a/library/cpp/messagebus/actor/what_thread_does.cpp +++ b/library/cpp/messagebus/actor/what_thread_does.cpp @@ -4,19 +4,19 @@ #include <util/system/tls.h> -Y_POD_STATIC_THREAD(const char*) -WhatThreadDoes; +Y_POD_STATIC_THREAD(const char*) +WhatThreadDoes; -const char* PushWhatThreadDoes(const char* what) { +const char* PushWhatThreadDoes(const char* what) { const char* r = NTSAN::RelaxedLoad(&WhatThreadDoes); NTSAN::RelaxedStore(&WhatThreadDoes, what); return r; } -void PopWhatThreadDoes(const char* prev) { +void PopWhatThreadDoes(const char* prev) { NTSAN::RelaxedStore(&WhatThreadDoes, prev); } -const char** WhatThreadDoesLocation() { +const char** WhatThreadDoesLocation() { return &WhatThreadDoes; } diff --git a/library/cpp/messagebus/actor/what_thread_does_guard.h b/library/cpp/messagebus/actor/what_thread_does_guard.h index f104e9e173..4fb0abef6d 100644 --- a/library/cpp/messagebus/actor/what_thread_does_guard.h +++ b/library/cpp/messagebus/actor/what_thread_does_guard.h @@ -4,37 +4,37 @@ template <class T> class TWhatThreadDoesAcquireGuard: public TNonCopyable { -public: - inline TWhatThreadDoesAcquireGuard(const T& t, const char* acquire) noexcept { - Init(&t, acquire); - } - - inline TWhatThreadDoesAcquireGuard(const T* t, const char* acquire) noexcept { - Init(t, acquire); - } - - inline ~TWhatThreadDoesAcquireGuard() { - Release(); - } - - inline void Release() noexcept { - if (WasAcquired()) { - const_cast<T*>(T_)->Release(); - T_ = nullptr; +public: + inline TWhatThreadDoesAcquireGuard(const T& t, const char* acquire) noexcept { + Init(&t, acquire); + } + + inline TWhatThreadDoesAcquireGuard(const T* t, const char* acquire) noexcept { + Init(t, acquire); + } + + inline ~TWhatThreadDoesAcquireGuard() { + Release(); + } + + inline void Release() noexcept { + if (WasAcquired()) { + const_cast<T*>(T_)->Release(); + T_ = nullptr; } - } + } - inline bool WasAcquired() const noexcept { - return T_ != nullptr; - } + inline bool WasAcquired() const noexcept { + return T_ != nullptr; + } -private: - inline void Init(const T* t, const char* acquire) noexcept { - T_ = const_cast<T*>(t); - TWhatThreadDoesPushPop pp(acquire); - T_->Acquire(); - } +private: + inline void Init(const T* t, const char* acquire) noexcept { + T_ = const_cast<T*>(t); + TWhatThreadDoesPushPop pp(acquire); + T_->Acquire(); + } -private: - T* T_; +private: + T* T_; }; diff --git a/library/cpp/messagebus/async_result.h b/library/cpp/messagebus/async_result.h index d24dde284a..3ae910796f 100644 --- a/library/cpp/messagebus/async_result.h +++ b/library/cpp/messagebus/async_result.h @@ -42,7 +42,7 @@ public: } template <typename TFunc> - void AndThen(const TFunc& onResult) { + void AndThen(const TFunc& onResult) { TGuard<TMutex> guard(Mutex); if (!!Result) { onResult(*Result); diff --git a/library/cpp/messagebus/base.h b/library/cpp/messagebus/base.h index 79fccc312e..78d9c39e4d 100644 --- a/library/cpp/messagebus/base.h +++ b/library/cpp/messagebus/base.h @@ -3,9 +3,9 @@ #include <util/system/defaults.h> namespace NBus { - /// millis since epoch - using TBusInstant = ui64; - /// returns time in milliseconds - TBusInstant Now(); + /// millis since epoch + using TBusInstant = ui64; + /// returns time in milliseconds + TBusInstant Now(); } diff --git a/library/cpp/messagebus/cc_semaphore.h b/library/cpp/messagebus/cc_semaphore.h index 0df8a3d664..34ec933184 100644 --- a/library/cpp/messagebus/cc_semaphore.h +++ b/library/cpp/messagebus/cc_semaphore.h @@ -25,9 +25,9 @@ public: } } - bool IsLocked() { - return Latch.IsLocked(); - } + bool IsLocked() { + return Latch.IsLocked(); + } private: TThis* GetThis() { diff --git a/library/cpp/messagebus/cc_semaphore_ut.cpp b/library/cpp/messagebus/cc_semaphore_ut.cpp index 206bb7c96a..0788e763df 100644 --- a/library/cpp/messagebus/cc_semaphore_ut.cpp +++ b/library/cpp/messagebus/cc_semaphore_ut.cpp @@ -5,13 +5,13 @@ #include <util/system/atomic.h> namespace { - struct TTestSemaphore: public TComplexConditionSemaphore<TTestSemaphore> { + struct TTestSemaphore: public TComplexConditionSemaphore<TTestSemaphore> { TAtomic Current; - TTestSemaphore() - : Current(0) - { - } + TTestSemaphore() + : Current(0) + { + } bool TryWait() { return AtomicGet(Current) > 0; diff --git a/library/cpp/messagebus/config/defs.h b/library/cpp/messagebus/config/defs.h index 92b1df9969..ad644cd279 100644 --- a/library/cpp/messagebus/config/defs.h +++ b/library/cpp/messagebus/config/defs.h @@ -1,5 +1,5 @@ #pragma once - + // unique tag to fix pragma once gcc glueing: ./library/cpp/messagebus/defs.h #include "codegen.h" @@ -18,65 +18,65 @@ class TCrawlerModule; struct TDebugReceiverHandler; namespace NBus { - namespace NPrivate { - class TAcceptor; - struct TBusSessionImpl; - class TRemoteServerSession; - class TRemoteClientSession; - class TRemoteConnection; - class TRemoteServerConnection; - class TRemoteClientConnection; - class TBusSyncSourceSessionImpl; + namespace NPrivate { + class TAcceptor; + struct TBusSessionImpl; + class TRemoteServerSession; + class TRemoteClientSession; + class TRemoteConnection; + class TRemoteServerConnection; + class TRemoteClientConnection; + class TBusSyncSourceSessionImpl; - struct TBusMessagePtrAndHeader; + struct TBusMessagePtrAndHeader; - struct TSessionDumpStatus; + struct TSessionDumpStatus; - struct TClientRequestImpl; + struct TClientRequestImpl; - } + } - class TBusSession; - struct TBusServerSession; - struct TBusClientSession; - class TBusProtocol; - class TBusMessage; - class TBusMessageConnection; - class TBusMessageQueue; - class TBusLocator; - struct TBusQueueConfig; - struct TBusSessionConfig; - struct TBusHeader; + class TBusSession; + struct TBusServerSession; + struct TBusClientSession; + class TBusProtocol; + class TBusMessage; + class TBusMessageConnection; + class TBusMessageQueue; + class TBusLocator; + struct TBusQueueConfig; + struct TBusSessionConfig; + struct TBusHeader; - class IThreadHandler; + class IThreadHandler; - using TBusKey = ui64; - using TBusMessageList = TList<TBusMessage*>; - using TBusKeyVec = TVector<std::pair<TBusKey, TBusKey>>; + using TBusKey = ui64; + using TBusMessageList = TList<TBusMessage*>; + using TBusKeyVec = TVector<std::pair<TBusKey, TBusKey>>; - using TBusMessageQueuePtr = TIntrusivePtr<TBusMessageQueue>; + using TBusMessageQueuePtr = TIntrusivePtr<TBusMessageQueue>; - class TBusModule; + class TBusModule; - using TBusData = TString; - using TBusService = const char*; + using TBusData = TString; + using TBusService = const char*; -#define YBUS_KEYMIN TBusKey(0L) -#define YBUS_KEYMAX TBusKey(-1L) -#define YBUS_KEYLOCAL TBusKey(7L) +#define YBUS_KEYMIN TBusKey(0L) +#define YBUS_KEYMAX TBusKey(-1L) +#define YBUS_KEYLOCAL TBusKey(7L) #define YBUS_KEYINVALID TBusKey(99999999L) - // Check that generated id is valid for remote message - inline bool IsBusKeyValid(TBusKey key) { - return key != YBUS_KEYINVALID && key != YBUS_KEYMAX && key > YBUS_KEYLOCAL; - } + // Check that generated id is valid for remote message + inline bool IsBusKeyValid(TBusKey key) { + return key != YBUS_KEYINVALID && key != YBUS_KEYMAX && key > YBUS_KEYLOCAL; + } -#define YBUS_VERSION 0 +#define YBUS_VERSION 0 -#define YBUS_INFINITE (1u << 30u) +#define YBUS_INFINITE (1u << 30u) -#define YBUS_STATUS_BASIC 0x0000 -#define YBUS_STATUS_CONNS 0x0001 +#define YBUS_STATUS_BASIC 0x0000 +#define YBUS_STATUS_CONNS 0x0001 #define YBUS_STATUS_INFLIGHT 0x0002 -} +} diff --git a/library/cpp/messagebus/config/netaddr.cpp b/library/cpp/messagebus/config/netaddr.cpp index 962ac538e2..73081240a0 100644 --- a/library/cpp/messagebus/config/netaddr.cpp +++ b/library/cpp/messagebus/config/netaddr.cpp @@ -5,175 +5,175 @@ #include <cstdlib> namespace NBus { - const char* ToCString(EIpVersion ipVersion) { - switch (ipVersion) { - case EIP_VERSION_ANY: - return "EIP_VERSION_ANY"; - case EIP_VERSION_4: - return "EIP_VERSION_4"; - case EIP_VERSION_6: - return "EIP_VERSION_6"; + const char* ToCString(EIpVersion ipVersion) { + switch (ipVersion) { + case EIP_VERSION_ANY: + return "EIP_VERSION_ANY"; + case EIP_VERSION_4: + return "EIP_VERSION_4"; + case EIP_VERSION_6: + return "EIP_VERSION_6"; + } + Y_FAIL(); + } + + int ToAddrFamily(EIpVersion ipVersion) { + switch (ipVersion) { + case EIP_VERSION_ANY: + return AF_UNSPEC; + case EIP_VERSION_4: + return AF_INET; + case EIP_VERSION_6: + return AF_INET6; + } + Y_FAIL(); + } + + class TNetworkAddressRef: private TNetworkAddress, public TAddrInfo { + public: + TNetworkAddressRef(const TNetworkAddress& na, const TAddrInfo& ai) + : TNetworkAddress(na) + , TAddrInfo(ai) + { + } + }; + + static bool Compare(const IRemoteAddr& l, const IRemoteAddr& r) noexcept { + if (l.Addr()->sa_family != r.Addr()->sa_family) { + return false; + } + + switch (l.Addr()->sa_family) { + case AF_INET: { + return memcmp(&(((const sockaddr_in*)l.Addr())->sin_addr), &(((const sockaddr_in*)r.Addr())->sin_addr), sizeof(in_addr)) == 0 && + ((const sockaddr_in*)l.Addr())->sin_port == ((const sockaddr_in*)r.Addr())->sin_port; + } + + case AF_INET6: { + return memcmp(&(((const sockaddr_in6*)l.Addr())->sin6_addr), &(((const sockaddr_in6*)r.Addr())->sin6_addr), sizeof(in6_addr)) == 0 && + ((const sockaddr_in6*)l.Addr())->sin6_port == ((const sockaddr_in6*)r.Addr())->sin6_port; + } } - Y_FAIL(); - } - - int ToAddrFamily(EIpVersion ipVersion) { - switch (ipVersion) { - case EIP_VERSION_ANY: - return AF_UNSPEC; - case EIP_VERSION_4: - return AF_INET; - case EIP_VERSION_6: - return AF_INET6; - } - Y_FAIL(); - } - - class TNetworkAddressRef: private TNetworkAddress, public TAddrInfo { - public: - TNetworkAddressRef(const TNetworkAddress& na, const TAddrInfo& ai) - : TNetworkAddress(na) - , TAddrInfo(ai) - { - } - }; - - static bool Compare(const IRemoteAddr& l, const IRemoteAddr& r) noexcept { - if (l.Addr()->sa_family != r.Addr()->sa_family) { - return false; - } - - switch (l.Addr()->sa_family) { - case AF_INET: { - return memcmp(&(((const sockaddr_in*)l.Addr())->sin_addr), &(((const sockaddr_in*)r.Addr())->sin_addr), sizeof(in_addr)) == 0 && - ((const sockaddr_in*)l.Addr())->sin_port == ((const sockaddr_in*)r.Addr())->sin_port; - } - - case AF_INET6: { - return memcmp(&(((const sockaddr_in6*)l.Addr())->sin6_addr), &(((const sockaddr_in6*)r.Addr())->sin6_addr), sizeof(in6_addr)) == 0 && - ((const sockaddr_in6*)l.Addr())->sin6_port == ((const sockaddr_in6*)r.Addr())->sin6_port; - } - } - - return memcmp(l.Addr(), r.Addr(), Min<size_t>(l.Len(), r.Len())) == 0; - } - - TNetAddr::TNetAddr() - : Ptr(new TOpaqueAddr) - { - } - TNetAddr::TNetAddr(TAutoPtr<IRemoteAddr> addr) - : Ptr(addr) - { - Y_VERIFY(!!Ptr); - } - - namespace { - using namespace NAddr; - - const char* Describe(EIpVersion version) { - switch (version) { - case EIP_VERSION_4: - return "ipv4 address"; - case EIP_VERSION_6: - return "ipv6 address"; - case EIP_VERSION_ANY: - return "any address"; - default: - Y_FAIL("unreachable"); - } + return memcmp(l.Addr(), r.Addr(), Min<size_t>(l.Len(), r.Len())) == 0; + } + + TNetAddr::TNetAddr() + : Ptr(new TOpaqueAddr) + { + } + + TNetAddr::TNetAddr(TAutoPtr<IRemoteAddr> addr) + : Ptr(addr) + { + Y_VERIFY(!!Ptr); + } + + namespace { + using namespace NAddr; + + const char* Describe(EIpVersion version) { + switch (version) { + case EIP_VERSION_4: + return "ipv4 address"; + case EIP_VERSION_6: + return "ipv6 address"; + case EIP_VERSION_ANY: + return "any address"; + default: + Y_FAIL("unreachable"); + } } - TAutoPtr<IRemoteAddr> MakeAddress(const TNetworkAddress& na, EIpVersion requireVersion, EIpVersion preferVersion) { - TAutoPtr<IRemoteAddr> addr; - for (TNetworkAddress::TIterator it = na.Begin(); it != na.End(); ++it) { - if (IsFamilyAllowed(it->ai_family, requireVersion)) { - if (IsFamilyAllowed(it->ai_family, preferVersion)) { - return new TNetworkAddressRef(na, &*it); - } else if (!addr) { - addr.Reset(new TNetworkAddressRef(na, &*it)); - } + TAutoPtr<IRemoteAddr> MakeAddress(const TNetworkAddress& na, EIpVersion requireVersion, EIpVersion preferVersion) { + TAutoPtr<IRemoteAddr> addr; + for (TNetworkAddress::TIterator it = na.Begin(); it != na.End(); ++it) { + if (IsFamilyAllowed(it->ai_family, requireVersion)) { + if (IsFamilyAllowed(it->ai_family, preferVersion)) { + return new TNetworkAddressRef(na, &*it); + } else if (!addr) { + addr.Reset(new TNetworkAddressRef(na, &*it)); + } } } - return addr; + return addr; } - TAutoPtr<IRemoteAddr> MakeAddress(TStringBuf host, int port, EIpVersion requireVersion, EIpVersion preferVersion) { - TString hostString(host); - TNetworkAddress na(hostString, port); - return MakeAddress(na, requireVersion, preferVersion); - } - TAutoPtr<IRemoteAddr> MakeAddress(const char* hostPort, EIpVersion requireVersion, EIpVersion preferVersion) { - const char* portStr = strchr(hostPort, ':'); - if (!portStr) { - ythrow TNetAddr::TError() << "port not specified in " << hostPort; - } - int port = atoi(portStr + 1); - TNetworkAddress na(TString(hostPort, portStr), port); - return MakeAddress(na, requireVersion, preferVersion); + TAutoPtr<IRemoteAddr> MakeAddress(TStringBuf host, int port, EIpVersion requireVersion, EIpVersion preferVersion) { + TString hostString(host); + TNetworkAddress na(hostString, port); + return MakeAddress(na, requireVersion, preferVersion); + } + TAutoPtr<IRemoteAddr> MakeAddress(const char* hostPort, EIpVersion requireVersion, EIpVersion preferVersion) { + const char* portStr = strchr(hostPort, ':'); + if (!portStr) { + ythrow TNetAddr::TError() << "port not specified in " << hostPort; + } + int port = atoi(portStr + 1); + TNetworkAddress na(TString(hostPort, portStr), port); + return MakeAddress(na, requireVersion, preferVersion); + } + } + + TNetAddr::TNetAddr(const char* hostPort, EIpVersion requireVersion /*= EIP_VERSION_ANY*/, EIpVersion preferVersion /*= EIP_VERSION_ANY*/) + : Ptr(MakeAddress(hostPort, requireVersion, preferVersion)) + { + if (!Ptr) { + ythrow TNetAddr::TError() << "cannot resolve " << hostPort << " into " << Describe(requireVersion); + } + } + + TNetAddr::TNetAddr(TStringBuf host, int port, EIpVersion requireVersion /*= EIP_VERSION_ANY*/, EIpVersion preferVersion /*= EIP_VERSION_ANY*/) + : Ptr(MakeAddress(host, port, requireVersion, preferVersion)) + { + if (!Ptr) { + ythrow TNetAddr::TError() << "cannot resolve " << host << ":" << port << " into " << Describe(requireVersion); } } - TNetAddr::TNetAddr(const char* hostPort, EIpVersion requireVersion /*= EIP_VERSION_ANY*/, EIpVersion preferVersion /*= EIP_VERSION_ANY*/) - : Ptr(MakeAddress(hostPort, requireVersion, preferVersion)) - { - if (!Ptr) { - ythrow TNetAddr::TError() << "cannot resolve " << hostPort << " into " << Describe(requireVersion); - } + TNetAddr::TNetAddr(const TNetworkAddress& na, EIpVersion requireVersion /*= EIP_VERSION_ANY*/, EIpVersion preferVersion /*= EIP_VERSION_ANY*/) + : Ptr(MakeAddress(na, requireVersion, preferVersion)) + { + if (!Ptr) { + ythrow TNetAddr::TError() << "cannot resolve into " << Describe(requireVersion); + } } - TNetAddr::TNetAddr(TStringBuf host, int port, EIpVersion requireVersion /*= EIP_VERSION_ANY*/, EIpVersion preferVersion /*= EIP_VERSION_ANY*/) - : Ptr(MakeAddress(host, port, requireVersion, preferVersion)) - { - if (!Ptr) { - ythrow TNetAddr::TError() << "cannot resolve " << host << ":" << port << " into " << Describe(requireVersion); - } + TNetAddr::TNetAddr(const TNetworkAddress& na, const TAddrInfo& ai) + : Ptr(new TNetworkAddressRef(na, ai)) + { } - TNetAddr::TNetAddr(const TNetworkAddress& na, EIpVersion requireVersion /*= EIP_VERSION_ANY*/, EIpVersion preferVersion /*= EIP_VERSION_ANY*/) - : Ptr(MakeAddress(na, requireVersion, preferVersion)) - { - if (!Ptr) { - ythrow TNetAddr::TError() << "cannot resolve into " << Describe(requireVersion); - } + const sockaddr* TNetAddr::Addr() const { + return Ptr->Addr(); } - TNetAddr::TNetAddr(const TNetworkAddress& na, const TAddrInfo& ai) - : Ptr(new TNetworkAddressRef(na, ai)) - { - } + socklen_t TNetAddr::Len() const { + return Ptr->Len(); + } - const sockaddr* TNetAddr::Addr() const { - return Ptr->Addr(); - } - - socklen_t TNetAddr::Len() const { - return Ptr->Len(); - } - - int TNetAddr::GetPort() const { - switch (Ptr->Addr()->sa_family) { - case AF_INET: - return InetToHost(((sockaddr_in*)Ptr->Addr())->sin_port); - case AF_INET6: - return InetToHost(((sockaddr_in6*)Ptr->Addr())->sin6_port); - default: - Y_FAIL("unknown AF: %d", (int)Ptr->Addr()->sa_family); - throw 1; - } - } + int TNetAddr::GetPort() const { + switch (Ptr->Addr()->sa_family) { + case AF_INET: + return InetToHost(((sockaddr_in*)Ptr->Addr())->sin_port); + case AF_INET6: + return InetToHost(((sockaddr_in6*)Ptr->Addr())->sin6_port); + default: + Y_FAIL("unknown AF: %d", (int)Ptr->Addr()->sa_family); + throw 1; + } + } - bool TNetAddr::IsIpv4() const { - return Ptr->Addr()->sa_family == AF_INET; - } + bool TNetAddr::IsIpv4() const { + return Ptr->Addr()->sa_family == AF_INET; + } - bool TNetAddr::IsIpv6() const { - return Ptr->Addr()->sa_family == AF_INET6; + bool TNetAddr::IsIpv6() const { + return Ptr->Addr()->sa_family == AF_INET6; } - bool TNetAddr::operator==(const TNetAddr& rhs) const { - return Ptr == rhs.Ptr || Compare(*Ptr, *rhs.Ptr); - } + bool TNetAddr::operator==(const TNetAddr& rhs) const { + return Ptr == rhs.Ptr || Compare(*Ptr, *rhs.Ptr); + } } diff --git a/library/cpp/messagebus/config/netaddr.h b/library/cpp/messagebus/config/netaddr.h index b79c0cc355..839f3d8f5d 100644 --- a/library/cpp/messagebus/config/netaddr.h +++ b/library/cpp/messagebus/config/netaddr.h @@ -8,79 +8,79 @@ #include <util/network/address.h> namespace NBus { - using namespace NAddr; - - /// IP protocol version. - enum EIpVersion { - EIP_VERSION_4 = 1, - EIP_VERSION_6 = 2, - EIP_VERSION_ANY = EIP_VERSION_4 | EIP_VERSION_6, - }; - - inline bool IsFamilyAllowed(ui16 sa_family, EIpVersion ipVersion) { - if (ipVersion == EIP_VERSION_4 && sa_family != AF_INET) { - return false; - } - if (ipVersion == EIP_VERSION_6 && sa_family != AF_INET6) { - return false; - } - return true; + using namespace NAddr; + + /// IP protocol version. + enum EIpVersion { + EIP_VERSION_4 = 1, + EIP_VERSION_6 = 2, + EIP_VERSION_ANY = EIP_VERSION_4 | EIP_VERSION_6, + }; + + inline bool IsFamilyAllowed(ui16 sa_family, EIpVersion ipVersion) { + if (ipVersion == EIP_VERSION_4 && sa_family != AF_INET) { + return false; + } + if (ipVersion == EIP_VERSION_6 && sa_family != AF_INET6) { + return false; + } + return true; } - const char* ToCString(EIpVersion); - int ToAddrFamily(EIpVersion); - - /// Hold referenced pointer to address description structure (ex. sockaddr_storage) - /// It's make possible to work with IPv4 / IPv6 addresses simultaneously - class TNetAddr: public IRemoteAddr { - public: - class TError: public yexception { - }; - - TNetAddr(); - TNetAddr(TAutoPtr<IRemoteAddr> addr); - TNetAddr(const char* hostPort, EIpVersion requireVersion = EIP_VERSION_ANY, EIpVersion preferVersion = EIP_VERSION_ANY); - TNetAddr(TStringBuf host, int port, EIpVersion requireVersion = EIP_VERSION_ANY, EIpVersion preferVersion = EIP_VERSION_ANY); - TNetAddr(const TNetworkAddress& na, EIpVersion requireVersion = EIP_VERSION_ANY, EIpVersion preferVersion = EIP_VERSION_ANY); - TNetAddr(const TNetworkAddress& na, const TAddrInfo& ai); - - bool operator==(const TNetAddr&) const; - bool operator!=(const TNetAddr& other) const { - return !(*this == other); - } - inline explicit operator bool() const noexcept { - return !!Ptr; - } - - const sockaddr* Addr() const override; - socklen_t Len() const override; - - bool IsIpv4() const; - bool IsIpv6() const; - int GetPort() const; - - private: - TAtomicSharedPtr<IRemoteAddr> Ptr; - }; - - using TSockAddrInVector = TVector<TNetAddr>; - - struct TNetAddrHostPortHash { - inline size_t operator()(const TNetAddr& a) const { - const sockaddr* s = a.Addr(); - const sockaddr_in* const sa = reinterpret_cast<const sockaddr_in*>(s); - const sockaddr_in6* const sa6 = reinterpret_cast<const sockaddr_in6*>(s); - - switch (s->sa_family) { - case AF_INET: + const char* ToCString(EIpVersion); + int ToAddrFamily(EIpVersion); + + /// Hold referenced pointer to address description structure (ex. sockaddr_storage) + /// It's make possible to work with IPv4 / IPv6 addresses simultaneously + class TNetAddr: public IRemoteAddr { + public: + class TError: public yexception { + }; + + TNetAddr(); + TNetAddr(TAutoPtr<IRemoteAddr> addr); + TNetAddr(const char* hostPort, EIpVersion requireVersion = EIP_VERSION_ANY, EIpVersion preferVersion = EIP_VERSION_ANY); + TNetAddr(TStringBuf host, int port, EIpVersion requireVersion = EIP_VERSION_ANY, EIpVersion preferVersion = EIP_VERSION_ANY); + TNetAddr(const TNetworkAddress& na, EIpVersion requireVersion = EIP_VERSION_ANY, EIpVersion preferVersion = EIP_VERSION_ANY); + TNetAddr(const TNetworkAddress& na, const TAddrInfo& ai); + + bool operator==(const TNetAddr&) const; + bool operator!=(const TNetAddr& other) const { + return !(*this == other); + } + inline explicit operator bool() const noexcept { + return !!Ptr; + } + + const sockaddr* Addr() const override; + socklen_t Len() const override; + + bool IsIpv4() const; + bool IsIpv6() const; + int GetPort() const; + + private: + TAtomicSharedPtr<IRemoteAddr> Ptr; + }; + + using TSockAddrInVector = TVector<TNetAddr>; + + struct TNetAddrHostPortHash { + inline size_t operator()(const TNetAddr& a) const { + const sockaddr* s = a.Addr(); + const sockaddr_in* const sa = reinterpret_cast<const sockaddr_in*>(s); + const sockaddr_in6* const sa6 = reinterpret_cast<const sockaddr_in6*>(s); + + switch (s->sa_family) { + case AF_INET: return CombineHashes<size_t>(ComputeHash(TStringBuf(reinterpret_cast<const char*>(&sa->sin_addr), sizeof(sa->sin_addr))), IntHashImpl(sa->sin_port)); - case AF_INET6: + case AF_INET6: return CombineHashes<size_t>(ComputeHash(TStringBuf(reinterpret_cast<const char*>(&sa6->sin6_addr), sizeof(sa6->sin6_addr))), IntHashImpl(sa6->sin6_port)); - } + } return ComputeHash(TStringBuf(reinterpret_cast<const char*>(s), a.Len())); } - }; + }; -} +} diff --git a/library/cpp/messagebus/config/session_config.cpp b/library/cpp/messagebus/config/session_config.cpp index fbbbb106c9..31c7dd29f2 100644 --- a/library/cpp/messagebus/config/session_config.cpp +++ b/library/cpp/messagebus/config/session_config.cpp @@ -13,8 +13,8 @@ TBusSessionConfig::TSecret::TSecret() TBusSessionConfig::TBusSessionConfig() : BUS_SESSION_CONFIG_MAP(STRUCT_FIELD_INIT, COMMA) -{ -} +{ +} TString TBusSessionConfig::PrintToString() const { TStringStream ss; @@ -40,10 +40,10 @@ static int ParseToSForMessageBus(const char* option) { return tos; } -template <class T> -static T ParseWithKmgSuffixT(const char* option) { +template <class T> +static T ParseWithKmgSuffixT(const char* option) { TStringBuf str(option); - T multiplier = 1; + T multiplier = 1; if (str.EndsWith('k')) { multiplier = 1024; str = str.Head(str.size() - 1); @@ -54,104 +54,104 @@ static T ParseWithKmgSuffixT(const char* option) { multiplier = 1024 * 1024 * 1024; str = str.Head(str.size() - 1); } - return FromString<T>(str) * multiplier; -} - -static ui64 ParseWithKmgSuffix(const char* option) { - return ParseWithKmgSuffixT<ui64>(option); -} - -static i64 ParseWithKmgSuffixS(const char* option) { - return ParseWithKmgSuffixT<i64>(option); + return FromString<T>(str) * multiplier; } +static ui64 ParseWithKmgSuffix(const char* option) { + return ParseWithKmgSuffixT<ui64>(option); +} + +static i64 ParseWithKmgSuffixS(const char* option) { + return ParseWithKmgSuffixT<i64>(option); +} + void TBusSessionConfig::ConfigureLastGetopt(NLastGetopt::TOpts& opts, const TString& prefix) { opts.AddLongOption(prefix + "total-timeout") - .RequiredArgument("MILLISECONDS") - .DefaultValue(ToString(TotalTimeout)) + .RequiredArgument("MILLISECONDS") + .DefaultValue(ToString(TotalTimeout)) .StoreMappedResultT<const char*>(&TotalTimeout, &ParseDurationForMessageBus); opts.AddLongOption(prefix + "connect-timeout") - .RequiredArgument("MILLISECONDS") - .DefaultValue(ToString(ConnectTimeout)) + .RequiredArgument("MILLISECONDS") + .DefaultValue(ToString(ConnectTimeout)) .StoreMappedResultT<const char*>(&ConnectTimeout, &ParseDurationForMessageBus); opts.AddLongOption(prefix + "send-timeout") - .RequiredArgument("MILLISECONDS") - .DefaultValue(ToString(SendTimeout)) + .RequiredArgument("MILLISECONDS") + .DefaultValue(ToString(SendTimeout)) .StoreMappedResultT<const char*>(&SendTimeout, &ParseDurationForMessageBus); opts.AddLongOption(prefix + "send-threshold") - .RequiredArgument("BYTES") - .DefaultValue(ToString(SendThreshold)) + .RequiredArgument("BYTES") + .DefaultValue(ToString(SendThreshold)) .StoreMappedResultT<const char*>(&SendThreshold, &ParseWithKmgSuffix); opts.AddLongOption(prefix + "max-in-flight") - .RequiredArgument("COUNT") - .DefaultValue(ToString(MaxInFlight)) + .RequiredArgument("COUNT") + .DefaultValue(ToString(MaxInFlight)) .StoreMappedResultT<const char*>(&MaxInFlight, &ParseWithKmgSuffix); opts.AddLongOption(prefix + "max-in-flight-by-size") - .RequiredArgument("BYTES") - .DefaultValue( - ToString(MaxInFlightBySize)) - .StoreMappedResultT<const char*>(&MaxInFlightBySize, &ParseWithKmgSuffixS); + .RequiredArgument("BYTES") + .DefaultValue( + ToString(MaxInFlightBySize)) + .StoreMappedResultT<const char*>(&MaxInFlightBySize, &ParseWithKmgSuffixS); opts.AddLongOption(prefix + "per-con-max-in-flight") - .RequiredArgument("COUNT") - .DefaultValue(ToString(PerConnectionMaxInFlight)) + .RequiredArgument("COUNT") + .DefaultValue(ToString(PerConnectionMaxInFlight)) .StoreMappedResultT<const char*>(&PerConnectionMaxInFlight, &ParseWithKmgSuffix); opts.AddLongOption(prefix + "per-con-max-in-flight-by-size") - .RequiredArgument("BYTES") - .DefaultValue( - ToString(PerConnectionMaxInFlightBySize)) + .RequiredArgument("BYTES") + .DefaultValue( + ToString(PerConnectionMaxInFlightBySize)) .StoreMappedResultT<const char*>(&PerConnectionMaxInFlightBySize, &ParseWithKmgSuffix); opts.AddLongOption(prefix + "default-buffer-size") - .RequiredArgument("BYTES") - .DefaultValue(ToString(DefaultBufferSize)) + .RequiredArgument("BYTES") + .DefaultValue(ToString(DefaultBufferSize)) .StoreMappedResultT<const char*>(&DefaultBufferSize, &ParseWithKmgSuffix); opts.AddLongOption(prefix + "max-buffer-size") - .RequiredArgument("BYTES") - .DefaultValue(ToString(MaxBufferSize)) + .RequiredArgument("BYTES") + .DefaultValue(ToString(MaxBufferSize)) .StoreMappedResultT<const char*>(&MaxBufferSize, &ParseWithKmgSuffix); opts.AddLongOption(prefix + "max-message-size") - .RequiredArgument("BYTES") - .DefaultValue(ToString(MaxMessageSize)) + .RequiredArgument("BYTES") + .DefaultValue(ToString(MaxMessageSize)) .StoreMappedResultT<const char*>(&MaxMessageSize, &ParseWithKmgSuffix); opts.AddLongOption(prefix + "socket-recv-buffer-size") - .RequiredArgument("BYTES") - .DefaultValue(ToString(SocketRecvBufferSize)) + .RequiredArgument("BYTES") + .DefaultValue(ToString(SocketRecvBufferSize)) .StoreMappedResultT<const char*>(&SocketRecvBufferSize, &ParseWithKmgSuffix); opts.AddLongOption(prefix + "socket-send-buffer-size") - .RequiredArgument("BYTES") - .DefaultValue(ToString(SocketSendBufferSize)) + .RequiredArgument("BYTES") + .DefaultValue(ToString(SocketSendBufferSize)) .StoreMappedResultT<const char*>(&SocketSendBufferSize, &ParseWithKmgSuffix); opts.AddLongOption(prefix + "socket-tos") .RequiredArgument("[0x00, 0xFF]") - .StoreMappedResultT<const char*>(&SocketToS, &ParseToSForMessageBus); - ; + .StoreMappedResultT<const char*>(&SocketToS, &ParseToSForMessageBus); + ; opts.AddLongOption(prefix + "tcp-cork") - .RequiredArgument("BOOL") - .DefaultValue(ToString(TcpCork)) + .RequiredArgument("BOOL") + .DefaultValue(ToString(TcpCork)) .StoreResult(&TcpCork); opts.AddLongOption(prefix + "cork") - .RequiredArgument("SECONDS") - .DefaultValue( - ToString(Cork.Seconds())) + .RequiredArgument("SECONDS") + .DefaultValue( + ToString(Cork.Seconds())) .StoreMappedResultT<const char*>(&Cork, &TDuration::Parse); opts.AddLongOption(prefix + "on-message-in-pool") - .RequiredArgument("BOOL") - .DefaultValue(ToString(ExecuteOnMessageInWorkerPool)) + .RequiredArgument("BOOL") + .DefaultValue(ToString(ExecuteOnMessageInWorkerPool)) .StoreResult(&ExecuteOnMessageInWorkerPool); opts.AddLongOption(prefix + "on-reply-in-pool") - .RequiredArgument("BOOL") - .DefaultValue(ToString(ExecuteOnReplyInWorkerPool)) + .RequiredArgument("BOOL") + .DefaultValue(ToString(ExecuteOnReplyInWorkerPool)) .StoreResult(&ExecuteOnReplyInWorkerPool); } diff --git a/library/cpp/messagebus/config/session_config.h b/library/cpp/messagebus/config/session_config.h index 84753350a9..db17018bbc 100644 --- a/library/cpp/messagebus/config/session_config.h +++ b/library/cpp/messagebus/config/session_config.h @@ -8,58 +8,58 @@ #include <util/generic/string.h> namespace NBus { -#define BUS_SESSION_CONFIG_MAP(XX, comma) \ - XX(Name, TString, "") \ - comma \ - XX(NumRetries, int, 0) comma \ - XX(RetryInterval, int, 1000) comma \ - XX(ReconnectWhenIdle, bool, false) comma \ - XX(MaxInFlight, i64, 1000) comma \ - XX(PerConnectionMaxInFlight, unsigned, 0) comma \ - XX(PerConnectionMaxInFlightBySize, unsigned, 0) comma \ - XX(MaxInFlightBySize, i64, -1) comma \ - XX(TotalTimeout, i64, 0) comma \ - XX(SendTimeout, i64, 0) comma \ - XX(ConnectTimeout, i64, 0) comma \ - XX(DefaultBufferSize, size_t, 10 * 1024) comma \ - XX(MaxBufferSize, size_t, 1024 * 1024) comma \ - XX(SocketRecvBufferSize, unsigned, 0) comma \ - XX(SocketSendBufferSize, unsigned, 0) comma \ - XX(SocketToS, int, -1) comma \ - XX(SendThreshold, size_t, 10 * 1024) comma \ - XX(Cork, TDuration, TDuration::Zero()) comma \ - XX(MaxMessageSize, unsigned, 26 << 20) comma \ - XX(TcpNoDelay, bool, false) comma \ - XX(TcpCork, bool, false) comma \ - XX(ExecuteOnMessageInWorkerPool, bool, true) comma \ - XX(ExecuteOnReplyInWorkerPool, bool, true) comma \ - XX(ReusePort, bool, false) comma \ - XX(ListenPort, unsigned, 0) /* TODO: server only */ +#define BUS_SESSION_CONFIG_MAP(XX, comma) \ + XX(Name, TString, "") \ + comma \ + XX(NumRetries, int, 0) comma \ + XX(RetryInterval, int, 1000) comma \ + XX(ReconnectWhenIdle, bool, false) comma \ + XX(MaxInFlight, i64, 1000) comma \ + XX(PerConnectionMaxInFlight, unsigned, 0) comma \ + XX(PerConnectionMaxInFlightBySize, unsigned, 0) comma \ + XX(MaxInFlightBySize, i64, -1) comma \ + XX(TotalTimeout, i64, 0) comma \ + XX(SendTimeout, i64, 0) comma \ + XX(ConnectTimeout, i64, 0) comma \ + XX(DefaultBufferSize, size_t, 10 * 1024) comma \ + XX(MaxBufferSize, size_t, 1024 * 1024) comma \ + XX(SocketRecvBufferSize, unsigned, 0) comma \ + XX(SocketSendBufferSize, unsigned, 0) comma \ + XX(SocketToS, int, -1) comma \ + XX(SendThreshold, size_t, 10 * 1024) comma \ + XX(Cork, TDuration, TDuration::Zero()) comma \ + XX(MaxMessageSize, unsigned, 26 << 20) comma \ + XX(TcpNoDelay, bool, false) comma \ + XX(TcpCork, bool, false) comma \ + XX(ExecuteOnMessageInWorkerPool, bool, true) comma \ + XX(ExecuteOnReplyInWorkerPool, bool, true) comma \ + XX(ReusePort, bool, false) comma \ + XX(ListenPort, unsigned, 0) /* TODO: server only */ - //////////////////////////////////////////////////////////////////// - /// \brief Configuration for client and server session - struct TBusSessionConfig { - BUS_SESSION_CONFIG_MAP(STRUCT_FIELD_GEN, ) + //////////////////////////////////////////////////////////////////// + /// \brief Configuration for client and server session + struct TBusSessionConfig { + BUS_SESSION_CONFIG_MAP(STRUCT_FIELD_GEN, ) - struct TSecret { - TDuration TimeoutPeriod; - TDuration StatusFlushPeriod; + struct TSecret { + TDuration TimeoutPeriod; + TDuration StatusFlushPeriod; - TSecret(); - }; + TSecret(); + }; - // secret options are available, but you shouldn't probably use them - TSecret Secret; + // secret options are available, but you shouldn't probably use them + TSecret Secret; - /// initialized with default settings - TBusSessionConfig(); + /// initialized with default settings + TBusSessionConfig(); - TString PrintToString() const; + TString PrintToString() const; - void ConfigureLastGetopt(NLastGetopt::TOpts&, const TString& prefix = "mb-"); - }; + void ConfigureLastGetopt(NLastGetopt::TOpts&, const TString& prefix = "mb-"); + }; - using TBusClientSessionConfig = TBusSessionConfig; - using TBusServerSessionConfig = TBusSessionConfig; + using TBusClientSessionConfig = TBusSessionConfig; + using TBusServerSessionConfig = TBusSessionConfig; } // NBus diff --git a/library/cpp/messagebus/connection.h b/library/cpp/messagebus/connection.h index b1df64ddc1..d88f6f753a 100644 --- a/library/cpp/messagebus/connection.h +++ b/library/cpp/messagebus/connection.h @@ -6,56 +6,56 @@ #include <util/generic/ptr.h> namespace NBus { - struct TBusClientConnection { - /// if you want to open connection early - virtual void OpenConnection() = 0; - - /// Send message to the destination - /// If addr is set then use it as destination. - /// Takes ownership of addr (see ClearState method). - virtual EMessageStatus SendMessage(TBusMessage* pMes, bool wait = false) = 0; - - virtual EMessageStatus SendMessageOneWay(TBusMessage* pMes, bool wait = false) = 0; - - /// Like SendMessage but cares about message - template <typename T /* <: TBusMessage */> - EMessageStatus SendMessageAutoPtr(const TAutoPtr<T>& mes, bool wait = false) { - EMessageStatus status = SendMessage(mes.Get(), wait); - if (status == MESSAGE_OK) + struct TBusClientConnection { + /// if you want to open connection early + virtual void OpenConnection() = 0; + + /// Send message to the destination + /// If addr is set then use it as destination. + /// Takes ownership of addr (see ClearState method). + virtual EMessageStatus SendMessage(TBusMessage* pMes, bool wait = false) = 0; + + virtual EMessageStatus SendMessageOneWay(TBusMessage* pMes, bool wait = false) = 0; + + /// Like SendMessage but cares about message + template <typename T /* <: TBusMessage */> + EMessageStatus SendMessageAutoPtr(const TAutoPtr<T>& mes, bool wait = false) { + EMessageStatus status = SendMessage(mes.Get(), wait); + if (status == MESSAGE_OK) Y_UNUSED(mes.Release()); - return status; - } - - /// Like SendMessageOneWay but cares about message - template <typename T /* <: TBusMessage */> - EMessageStatus SendMessageOneWayAutoPtr(const TAutoPtr<T>& mes, bool wait = false) { - EMessageStatus status = SendMessageOneWay(mes.Get(), wait); - if (status == MESSAGE_OK) + return status; + } + + /// Like SendMessageOneWay but cares about message + template <typename T /* <: TBusMessage */> + EMessageStatus SendMessageOneWayAutoPtr(const TAutoPtr<T>& mes, bool wait = false) { + EMessageStatus status = SendMessageOneWay(mes.Get(), wait); + if (status == MESSAGE_OK) Y_UNUSED(mes.Release()); - return status; - } + return status; + } - EMessageStatus SendMessageMove(TBusMessageAutoPtr message, bool wait = false) { - return SendMessageAutoPtr(message, wait); - } + EMessageStatus SendMessageMove(TBusMessageAutoPtr message, bool wait = false) { + return SendMessageAutoPtr(message, wait); + } - EMessageStatus SendMessageOneWayMove(TBusMessageAutoPtr message, bool wait = false) { - return SendMessageOneWayAutoPtr(message, wait); - } + EMessageStatus SendMessageOneWayMove(TBusMessageAutoPtr message, bool wait = false) { + return SendMessageOneWayAutoPtr(message, wait); + } - // TODO: implement similar one-way methods + // TODO: implement similar one-way methods - virtual ~TBusClientConnection() { - } + virtual ~TBusClientConnection() { + } }; - namespace NPrivate { - struct TBusClientConnectionPtrOps { - static void Ref(TBusClientConnection*); - static void UnRef(TBusClientConnection*); - }; - } + namespace NPrivate { + struct TBusClientConnectionPtrOps { + static void Ref(TBusClientConnection*); + static void UnRef(TBusClientConnection*); + }; + } - using TBusClientConnectionPtr = TIntrusivePtr<TBusClientConnection, NPrivate::TBusClientConnectionPtrOps>; + using TBusClientConnectionPtr = TIntrusivePtr<TBusClientConnection, NPrivate::TBusClientConnectionPtrOps>; } diff --git a/library/cpp/messagebus/coreconn.cpp b/library/cpp/messagebus/coreconn.cpp index d9411bb5db..edfb174a55 100644 --- a/library/cpp/messagebus/coreconn.cpp +++ b/library/cpp/messagebus/coreconn.cpp @@ -4,27 +4,27 @@ #include <util/datetime/base.h> #include <util/generic/yexception.h> -#include <util/network/socket.h> +#include <util/network/socket.h> #include <util/string/util.h> -#include <util/system/thread.h> - +#include <util/system/thread.h> + namespace NBus { - TBusInstant Now() { - return millisec(); - } + TBusInstant Now() { + return millisec(); + } - EIpVersion MakeIpVersion(bool allowIpv4, bool allowIpv6) { - if (allowIpv4) { - if (allowIpv6) { - return EIP_VERSION_ANY; - } else { - return EIP_VERSION_4; - } - } else if (allowIpv6) { - return EIP_VERSION_6; - } + EIpVersion MakeIpVersion(bool allowIpv4, bool allowIpv6) { + if (allowIpv4) { + if (allowIpv6) { + return EIP_VERSION_ANY; + } else { + return EIP_VERSION_4; + } + } else if (allowIpv6) { + return EIP_VERSION_6; + } - ythrow yexception() << "Neither of IPv4/IPv6 is allowed."; + ythrow yexception() << "Neither of IPv4/IPv6 is allowed."; } } diff --git a/library/cpp/messagebus/coreconn.h b/library/cpp/messagebus/coreconn.h index fca228d82e..4d50e0b5fb 100644 --- a/library/cpp/messagebus/coreconn.h +++ b/library/cpp/messagebus/coreconn.h @@ -3,65 +3,65 @@ ////////////////////////////////////////////////////////////// /// \file /// \brief Definitions for asynchonous connection queue - + #include "base.h" #include "event_loop.h" #include "netaddr.h" #include <util/datetime/base.h> -#include <util/generic/algorithm.h> +#include <util/generic/algorithm.h> #include <util/generic/list.h> #include <util/generic/map.h> #include <util/generic/set.h> #include <util/generic/string.h> #include <util/generic/vector.h> #include <util/network/address.h> -#include <util/network/ip.h> +#include <util/network/ip.h> #include <util/network/poller.h> #include <util/string/util.h> #include <util/system/condvar.h> -#include <util/system/mutex.h> +#include <util/system/mutex.h> #include <util/system/thread.h> #include <util/thread/lfqueue.h> - + #include <deque> #include <utility> #ifdef NO_ERROR -#undef NO_ERROR +#undef NO_ERROR #endif #define BUS_WORKER_CONDVAR //#define BUS_WORKER_MIXED -namespace NBus { - class TBusConnection; - class TBusConnectionFactory; - class TBusServerFactory; - - using TBusConnectionList = TList<TBusConnection*>; +namespace NBus { + class TBusConnection; + class TBusConnectionFactory; + class TBusServerFactory; - /// @throw yexception - EIpVersion MakeIpVersion(bool allowIpv4, bool allowIpv6); + using TBusConnectionList = TList<TBusConnection*>; - inline bool WouldBlock() { - int syserr = LastSystemError(); - return syserr == EAGAIN || syserr == EINPROGRESS || syserr == EWOULDBLOCK || syserr == EINTR; - } + /// @throw yexception + EIpVersion MakeIpVersion(bool allowIpv4, bool allowIpv6); - class TBusSession; + inline bool WouldBlock() { + int syserr = LastSystemError(); + return syserr == EAGAIN || syserr == EINPROGRESS || syserr == EWOULDBLOCK || syserr == EINTR; + } - struct TMaxConnectedException: public yexception { - TMaxConnectedException(unsigned maxConnect) { - yexception& exc = *this; + class TBusSession; + + struct TMaxConnectedException: public yexception { + TMaxConnectedException(unsigned maxConnect) { + yexception& exc = *this; exc << TStringBuf("Exceeded maximum number of outstanding connections: "); - exc << maxConnect; - } - }; + exc << maxConnect; + } + }; - enum EPollType { - POLL_READ, - POLL_WRITE - }; + enum EPollType { + POLL_READ, + POLL_WRITE + }; } diff --git a/library/cpp/messagebus/debug_receiver/debug_receiver_handler.cpp b/library/cpp/messagebus/debug_receiver/debug_receiver_handler.cpp index 05f99e94ca..7a535290ab 100644 --- a/library/cpp/messagebus/debug_receiver/debug_receiver_handler.cpp +++ b/library/cpp/messagebus/debug_receiver/debug_receiver_handler.cpp @@ -5,16 +5,16 @@ #include <util/generic/cast.h> #include <util/string/printf.h> -void TDebugReceiverHandler::OnError(TAutoPtr<NBus::TBusMessage>, NBus::EMessageStatus status) { +void TDebugReceiverHandler::OnError(TAutoPtr<NBus::TBusMessage>, NBus::EMessageStatus status) { Cerr << "error " << status << "\n"; } -void TDebugReceiverHandler::OnMessage(NBus::TOnMessageContext& message) { +void TDebugReceiverHandler::OnMessage(NBus::TOnMessageContext& message) { TDebugReceiverMessage* typedMessage = VerifyDynamicCast<TDebugReceiverMessage*>(message.GetMessage()); Cerr << "type=" << typedMessage->GetHeader()->Type - << " size=" << typedMessage->GetHeader()->Size - << " flags=" << Sprintf("0x%04x", (int)typedMessage->GetHeader()->FlagsInternal) - << "\n"; + << " size=" << typedMessage->GetHeader()->Size + << " flags=" << Sprintf("0x%04x", (int)typedMessage->GetHeader()->FlagsInternal) + << "\n"; message.ForgetRequest(); } diff --git a/library/cpp/messagebus/debug_receiver/debug_receiver_handler.h b/library/cpp/messagebus/debug_receiver/debug_receiver_handler.h index 0aed6b9984..a01140e06c 100644 --- a/library/cpp/messagebus/debug_receiver/debug_receiver_handler.h +++ b/library/cpp/messagebus/debug_receiver/debug_receiver_handler.h @@ -2,7 +2,7 @@ #include <library/cpp/messagebus/ybus.h> -struct TDebugReceiverHandler: public NBus::IBusServerHandler { +struct TDebugReceiverHandler: public NBus::IBusServerHandler { NBus::TBusServerSession* ServerSession; void OnError(TAutoPtr<NBus::TBusMessage> pMessage, NBus::EMessageStatus status) override; diff --git a/library/cpp/messagebus/debug_receiver/debug_receiver_proto.cpp b/library/cpp/messagebus/debug_receiver/debug_receiver_proto.cpp index 0c74f9ecc3..3dcb949a20 100644 --- a/library/cpp/messagebus/debug_receiver/debug_receiver_proto.cpp +++ b/library/cpp/messagebus/debug_receiver/debug_receiver_proto.cpp @@ -7,11 +7,11 @@ TDebugReceiverProtocol::TDebugReceiverProtocol() { } -void TDebugReceiverProtocol::Serialize(const NBus::TBusMessage*, TBuffer&) { +void TDebugReceiverProtocol::Serialize(const NBus::TBusMessage*, TBuffer&) { Y_FAIL("it is receiver only"); } -TAutoPtr<NBus::TBusMessage> TDebugReceiverProtocol::Deserialize(ui16, TArrayRef<const char> payload) { +TAutoPtr<NBus::TBusMessage> TDebugReceiverProtocol::Deserialize(ui16, TArrayRef<const char> payload) { THolder<TDebugReceiverMessage> r(new TDebugReceiverMessage(ECreateUninitialized())); r->Payload.Append(payload.data(), payload.size()); diff --git a/library/cpp/messagebus/debug_receiver/debug_receiver_proto.h b/library/cpp/messagebus/debug_receiver/debug_receiver_proto.h index d34710dcf7..ed08d3cd9e 100644 --- a/library/cpp/messagebus/debug_receiver/debug_receiver_proto.h +++ b/library/cpp/messagebus/debug_receiver/debug_receiver_proto.h @@ -2,26 +2,26 @@ #include <library/cpp/messagebus/ybus.h> -struct TDebugReceiverMessage: public NBus::TBusMessage { +struct TDebugReceiverMessage: public NBus::TBusMessage { /// constructor to create messages on sending end - TDebugReceiverMessage(ui16 type) - : NBus::TBusMessage(type) - { - } + TDebugReceiverMessage(ui16 type) + : NBus::TBusMessage(type) + { + } /// constructor with serialzed data to examine the header - TDebugReceiverMessage(NBus::ECreateUninitialized) - : NBus::TBusMessage(NBus::ECreateUninitialized()) - { - } + TDebugReceiverMessage(NBus::ECreateUninitialized) + : NBus::TBusMessage(NBus::ECreateUninitialized()) + { + } TBuffer Payload; }; -struct TDebugReceiverProtocol: public NBus::TBusProtocol { +struct TDebugReceiverProtocol: public NBus::TBusProtocol { TDebugReceiverProtocol(); - void Serialize(const NBus::TBusMessage* mess, TBuffer& data) override; + void Serialize(const NBus::TBusMessage* mess, TBuffer& data) override; TAutoPtr<NBus::TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) override; }; diff --git a/library/cpp/messagebus/defs.h b/library/cpp/messagebus/defs.h index cb553acc45..7d7f34a743 100644 --- a/library/cpp/messagebus/defs.h +++ b/library/cpp/messagebus/defs.h @@ -1,4 +1,4 @@ #pragma once - + #include <library/cpp/messagebus/config/defs.h> diff --git a/library/cpp/messagebus/dummy_debugger.h b/library/cpp/messagebus/dummy_debugger.h index 89a4e18716..911d56ce73 100644 --- a/library/cpp/messagebus/dummy_debugger.h +++ b/library/cpp/messagebus/dummy_debugger.h @@ -3,7 +3,7 @@ #include <util/datetime/base.h> #include <util/stream/output.h> -#define MB_TRACE() \ - do { \ +#define MB_TRACE() \ + do { \ Cerr << TInstant::Now() << " " << __FILE__ << ":" << __LINE__ << " " << __FUNCTION__ << Endl; \ } while (false) diff --git a/library/cpp/messagebus/duration_histogram.cpp b/library/cpp/messagebus/duration_histogram.cpp index 32a0001d41..a2bbdef46a 100644 --- a/library/cpp/messagebus/duration_histogram.cpp +++ b/library/cpp/messagebus/duration_histogram.cpp @@ -47,7 +47,7 @@ namespace { }; } -TString TDurationHistogram::LabelBefore(unsigned i) { +TString TDurationHistogram::LabelBefore(unsigned i) { Y_VERIFY(i < Buckets); TDuration d = Singleton<TMarks>()->Marks[i]; @@ -65,7 +65,7 @@ TString TDurationHistogram::LabelBefore(unsigned i) { return ss.Str(); } -TString TDurationHistogram::PrintToString() const { +TString TDurationHistogram::PrintToString() const { TStringStream ss; for (auto time : Times) { ss << time << "\n"; diff --git a/library/cpp/messagebus/duration_histogram_ut.cpp b/library/cpp/messagebus/duration_histogram_ut.cpp index 01bcc095e9..006022216c 100644 --- a/library/cpp/messagebus/duration_histogram_ut.cpp +++ b/library/cpp/messagebus/duration_histogram_ut.cpp @@ -4,15 +4,15 @@ Y_UNIT_TEST_SUITE(TDurationHistogramTest) { Y_UNIT_TEST(BucketFor) { - UNIT_ASSERT_VALUES_EQUAL(0u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(0))); - UNIT_ASSERT_VALUES_EQUAL(0u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(1))); - UNIT_ASSERT_VALUES_EQUAL(0u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(900))); + UNIT_ASSERT_VALUES_EQUAL(0u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(0))); + UNIT_ASSERT_VALUES_EQUAL(0u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(1))); + UNIT_ASSERT_VALUES_EQUAL(0u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(900))); UNIT_ASSERT_VALUES_EQUAL(1u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(1500))); UNIT_ASSERT_VALUES_EQUAL(2u, TDurationHistogram::BucketFor(TDuration::MicroSeconds(2500))); unsigned sb = TDurationHistogram::SecondBoundary; - UNIT_ASSERT_VALUES_EQUAL(sb - 1, TDurationHistogram::BucketFor(TDuration::MilliSeconds(999))); + UNIT_ASSERT_VALUES_EQUAL(sb - 1, TDurationHistogram::BucketFor(TDuration::MilliSeconds(999))); UNIT_ASSERT_VALUES_EQUAL(sb, TDurationHistogram::BucketFor(TDuration::MilliSeconds(1000))); UNIT_ASSERT_VALUES_EQUAL(sb, TDurationHistogram::BucketFor(TDuration::MilliSeconds(1001))); diff --git a/library/cpp/messagebus/event_loop.cpp b/library/cpp/messagebus/event_loop.cpp index f685135bed..42477bf091 100644 --- a/library/cpp/messagebus/event_loop.cpp +++ b/library/cpp/messagebus/event_loop.cpp @@ -131,7 +131,7 @@ TEventLoop::TEventLoop(const char* name) { } -TEventLoop::~TEventLoop() { +TEventLoop::~TEventLoop() { } void TEventLoop::Run() { @@ -273,7 +273,7 @@ TEventLoop::TImpl::TImpl(const char* name) SetNonBlock(WakeupReadSocket, true); Poller.WaitRead(WakeupReadSocket, - reinterpret_cast<void*>(this)); + reinterpret_cast<void*>(this)); } void TEventLoop::TImpl::Run() { @@ -303,8 +303,8 @@ void TEventLoop::TImpl::Run() { s->CallHandler(); } - SOCKET socket = -1; - while (SocketsToRemove.Dequeue(&socket)) { + SOCKET socket = -1; + while (SocketsToRemove.Dequeue(&socket)) { TGuard<TMutex> guard(Mutex); Y_VERIFY(Data.erase(socket) == 1, "must be removed once"); } diff --git a/library/cpp/messagebus/event_loop.h b/library/cpp/messagebus/event_loop.h index d5b0a53b0c..a5d376b60a 100644 --- a/library/cpp/messagebus/event_loop.h +++ b/library/cpp/messagebus/event_loop.h @@ -7,10 +7,10 @@ namespace NEventLoop { struct IEventHandler - : public TAtomicRefCount<IEventHandler> { + : public TAtomicRefCount<IEventHandler> { virtual void HandleEvent(SOCKET socket, void* cookie) = 0; - virtual ~IEventHandler() { - } + virtual ~IEventHandler() { + } }; typedef TIntrusivePtr<IEventHandler> TEventHandlerPtr; @@ -20,7 +20,7 @@ namespace NEventLoop { // TODO: make TChannel itself a pointer // to avoid confusion with Drop and Unregister class TChannel - : public TAtomicRefCount<TChannel> { + : public TAtomicRefCount<TChannel> { public: ~TChannel(); diff --git a/library/cpp/messagebus/extra_ref.h b/library/cpp/messagebus/extra_ref.h index 2927123266..274dda1c71 100644 --- a/library/cpp/messagebus/extra_ref.h +++ b/library/cpp/messagebus/extra_ref.h @@ -6,10 +6,10 @@ class TExtraRef { TAtomic Holds; public: - TExtraRef() - : Holds(false) - { - } + TExtraRef() + : Holds(false) + { + } ~TExtraRef() { Y_VERIFY(!Holds); } diff --git a/library/cpp/messagebus/futex_like.cpp b/library/cpp/messagebus/futex_like.cpp index 7f965126db..1d6b105ba4 100644 --- a/library/cpp/messagebus/futex_like.cpp +++ b/library/cpp/messagebus/futex_like.cpp @@ -3,11 +3,11 @@ #ifdef _linux_ #include <sys/syscall.h> #include <linux/futex.h> - -#if !defined(SYS_futex) -#define SYS_futex __NR_futex -#endif + +#if !defined(SYS_futex) +#define SYS_futex __NR_futex #endif +#endif #include <errno.h> @@ -17,8 +17,8 @@ #ifdef _linux_ namespace { - int futex(int* uaddr, int op, int val, const struct timespec* timeout, - int* uaddr2, int val3) { + int futex(int* uaddr, int op, int val, const struct timespec* timeout, + int* uaddr2, int val3) { return syscall(SYS_futex, uaddr, op, val, timeout, uaddr2, val3); } } diff --git a/library/cpp/messagebus/futex_like.h b/library/cpp/messagebus/futex_like.h index 31d60c60f1..754d1a44c0 100644 --- a/library/cpp/messagebus/futex_like.h +++ b/library/cpp/messagebus/futex_like.h @@ -17,8 +17,8 @@ private: public: TFutexLike() : Value(0) - { - } + { + } int AddAndGet(int add) { #ifdef _linux_ diff --git a/library/cpp/messagebus/handler.cpp b/library/cpp/messagebus/handler.cpp index 333bd52934..eec3f3191d 100644 --- a/library/cpp/messagebus/handler.cpp +++ b/library/cpp/messagebus/handler.cpp @@ -6,31 +6,31 @@ using namespace NBus; using namespace NBus::NPrivate; -void IBusErrorHandler::OnError(TAutoPtr<TBusMessage> pMessage, EMessageStatus status) { - Y_UNUSED(pMessage); - Y_UNUSED(status); -} -void IBusServerHandler::OnSent(TAutoPtr<TBusMessage> pMessage) { - Y_UNUSED(pMessage); -} -void IBusClientHandler::OnMessageSent(TBusMessage* pMessage) { - Y_UNUSED(pMessage); -} -void IBusClientHandler::OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage) { - Y_UNUSED(pMessage); -} +void IBusErrorHandler::OnError(TAutoPtr<TBusMessage> pMessage, EMessageStatus status) { + Y_UNUSED(pMessage); + Y_UNUSED(status); +} +void IBusServerHandler::OnSent(TAutoPtr<TBusMessage> pMessage) { + Y_UNUSED(pMessage); +} +void IBusClientHandler::OnMessageSent(TBusMessage* pMessage) { + Y_UNUSED(pMessage); +} +void IBusClientHandler::OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage) { + Y_UNUSED(pMessage); +} -void IBusClientHandler::OnClientConnectionEvent(const TClientConnectionEvent&) { -} +void IBusClientHandler::OnClientConnectionEvent(const TClientConnectionEvent&) { +} -void TOnMessageContext::ForgetRequest() { +void TOnMessageContext::ForgetRequest() { Session->ForgetRequest(Ident); } -TNetAddr TOnMessageContext::GetPeerAddrNetAddr() const { +TNetAddr TOnMessageContext::GetPeerAddrNetAddr() const { return Ident.GetNetAddr(); } bool TOnMessageContext::IsConnectionAlive() const { - return !!Ident.Connection && Ident.Connection->IsAlive(); + return !!Ident.Connection && Ident.Connection->IsAlive(); } diff --git a/library/cpp/messagebus/handler.h b/library/cpp/messagebus/handler.h index 60002c68a6..211b2f5bca 100644 --- a/library/cpp/messagebus/handler.h +++ b/library/cpp/messagebus/handler.h @@ -9,127 +9,127 @@ #include <util/generic/noncopyable.h> namespace NBus { - ///////////////////////////////////////////////////////////////// - /// \brief Interface to message bus handler - - struct IBusErrorHandler { - friend struct ::NBus::NPrivate::TBusSessionImpl; - - private: - TUseAfterFreeChecker UseAfterFreeChecker; - TUseCountChecker UseCountChecker; - - public: - /// called when message or reply can't be delivered - virtual void OnError(TAutoPtr<TBusMessage> pMessage, EMessageStatus status); - - virtual ~IBusErrorHandler() { - } - }; - - class TClientConnectionEvent { - public: - enum EType { - CONNECTED, - DISCONNECTED, - }; - - private: - EType Type; - ui64 Id; - TNetAddr Addr; - - public: - TClientConnectionEvent(EType type, ui64 id, TNetAddr addr) - : Type(type) - , Id(id) - , Addr(addr) - { - } - - EType GetType() const { - return Type; - } - ui64 GetId() const { - return Id; - } - TNetAddr GetAddr() const { - return Addr; - } - }; - - class TOnMessageContext : TNonCopyable { - private: - THolder<TBusMessage> Message; - TBusIdentity Ident; - // TODO: we don't need to store session, we have connection in ident - TBusServerSession* Session; - - public: - TOnMessageContext() - : Session() - { - } - TOnMessageContext(TAutoPtr<TBusMessage> message, TBusIdentity& ident, TBusServerSession* session) - : Message(message) - , Session(session) - { - Ident.Swap(ident); - } - - bool IsInWork() const { - return Ident.IsInWork(); - } - - bool operator!() const { - return !IsInWork(); - } - - TBusMessage* GetMessage() { - return Message.Get(); - } - - TBusMessage* ReleaseMessage() { - return Message.Release(); - } - - TBusServerSession* GetSession() { - return Session; - } - - template <typename U /* <: TBusMessage */> - EMessageStatus SendReplyAutoPtr(TAutoPtr<U>& rep); - - EMessageStatus SendReplyMove(TBusMessageAutoPtr response); - - void AckMessage(TBusIdentity& ident); - - void ForgetRequest(); - - void Swap(TOnMessageContext& that) { - DoSwap(Message, that.Message); - Ident.Swap(that.Ident); - DoSwap(Session, that.Session); - } - - TNetAddr GetPeerAddrNetAddr() const; - - bool IsConnectionAlive() const; - }; - - struct IBusServerHandler: public IBusErrorHandler { - virtual void OnMessage(TOnMessageContext& onMessage) = 0; - /// called when reply has been sent from destination - virtual void OnSent(TAutoPtr<TBusMessage> pMessage); - }; - - struct IBusClientHandler: public IBusErrorHandler { - /// called on source when reply arrives from destination - virtual void OnReply(TAutoPtr<TBusMessage> pMessage, TAutoPtr<TBusMessage> pReply) = 0; - /// called when client side message has gone into wire, place to call AckMessage() - virtual void OnMessageSent(TBusMessage* pMessage); - virtual void OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage); - virtual void OnClientConnectionEvent(const TClientConnectionEvent&); + ///////////////////////////////////////////////////////////////// + /// \brief Interface to message bus handler + + struct IBusErrorHandler { + friend struct ::NBus::NPrivate::TBusSessionImpl; + + private: + TUseAfterFreeChecker UseAfterFreeChecker; + TUseCountChecker UseCountChecker; + + public: + /// called when message or reply can't be delivered + virtual void OnError(TAutoPtr<TBusMessage> pMessage, EMessageStatus status); + + virtual ~IBusErrorHandler() { + } + }; + + class TClientConnectionEvent { + public: + enum EType { + CONNECTED, + DISCONNECTED, + }; + + private: + EType Type; + ui64 Id; + TNetAddr Addr; + + public: + TClientConnectionEvent(EType type, ui64 id, TNetAddr addr) + : Type(type) + , Id(id) + , Addr(addr) + { + } + + EType GetType() const { + return Type; + } + ui64 GetId() const { + return Id; + } + TNetAddr GetAddr() const { + return Addr; + } }; -} + class TOnMessageContext : TNonCopyable { + private: + THolder<TBusMessage> Message; + TBusIdentity Ident; + // TODO: we don't need to store session, we have connection in ident + TBusServerSession* Session; + + public: + TOnMessageContext() + : Session() + { + } + TOnMessageContext(TAutoPtr<TBusMessage> message, TBusIdentity& ident, TBusServerSession* session) + : Message(message) + , Session(session) + { + Ident.Swap(ident); + } + + bool IsInWork() const { + return Ident.IsInWork(); + } + + bool operator!() const { + return !IsInWork(); + } + + TBusMessage* GetMessage() { + return Message.Get(); + } + + TBusMessage* ReleaseMessage() { + return Message.Release(); + } + + TBusServerSession* GetSession() { + return Session; + } + + template <typename U /* <: TBusMessage */> + EMessageStatus SendReplyAutoPtr(TAutoPtr<U>& rep); + + EMessageStatus SendReplyMove(TBusMessageAutoPtr response); + + void AckMessage(TBusIdentity& ident); + + void ForgetRequest(); + + void Swap(TOnMessageContext& that) { + DoSwap(Message, that.Message); + Ident.Swap(that.Ident); + DoSwap(Session, that.Session); + } + + TNetAddr GetPeerAddrNetAddr() const; + + bool IsConnectionAlive() const; + }; + + struct IBusServerHandler: public IBusErrorHandler { + virtual void OnMessage(TOnMessageContext& onMessage) = 0; + /// called when reply has been sent from destination + virtual void OnSent(TAutoPtr<TBusMessage> pMessage); + }; + + struct IBusClientHandler: public IBusErrorHandler { + /// called on source when reply arrives from destination + virtual void OnReply(TAutoPtr<TBusMessage> pMessage, TAutoPtr<TBusMessage> pReply) = 0; + /// called when client side message has gone into wire, place to call AckMessage() + virtual void OnMessageSent(TBusMessage* pMessage); + virtual void OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage); + virtual void OnClientConnectionEvent(const TClientConnectionEvent&); + }; + +} diff --git a/library/cpp/messagebus/handler_impl.h b/library/cpp/messagebus/handler_impl.h index 6593f04cc3..94088cfba0 100644 --- a/library/cpp/messagebus/handler_impl.h +++ b/library/cpp/messagebus/handler_impl.h @@ -5,19 +5,19 @@ #include "session.h" namespace NBus { - template <typename U /* <: TBusMessage */> - EMessageStatus TOnMessageContext::SendReplyAutoPtr(TAutoPtr<U>& response) { - return Session->SendReplyAutoPtr(Ident, response); - } + template <typename U /* <: TBusMessage */> + EMessageStatus TOnMessageContext::SendReplyAutoPtr(TAutoPtr<U>& response) { + return Session->SendReplyAutoPtr(Ident, response); + } - inline EMessageStatus TOnMessageContext::SendReplyMove(TBusMessageAutoPtr response) { - return SendReplyAutoPtr(response); - } + inline EMessageStatus TOnMessageContext::SendReplyMove(TBusMessageAutoPtr response) { + return SendReplyAutoPtr(response); + } - inline void TOnMessageContext::AckMessage(TBusIdentity& ident) { - Y_VERIFY(Ident.LocalFlags == NPrivate::MESSAGE_IN_WORK); - Y_VERIFY(ident.LocalFlags == 0); - Ident.Swap(ident); - } + inline void TOnMessageContext::AckMessage(TBusIdentity& ident) { + Y_VERIFY(Ident.LocalFlags == NPrivate::MESSAGE_IN_WORK); + Y_VERIFY(ident.LocalFlags == 0); + Ident.Swap(ident); + } } diff --git a/library/cpp/messagebus/hash.h b/library/cpp/messagebus/hash.h index cc1b136a86..d20bb370b7 100644 --- a/library/cpp/messagebus/hash.h +++ b/library/cpp/messagebus/hash.h @@ -3,17 +3,17 @@ #include <util/str_stl.h> #include <util/digest/numeric.h> -namespace NBus { - namespace NPrivate { - template <typename T> - size_t Hash(const T& val) { - return THash<T>()(val); - } +namespace NBus { + namespace NPrivate { + template <typename T> + size_t Hash(const T& val) { + return THash<T>()(val); + } - template <typename T, typename U> - size_t HashValues(const T& a, const U& b) { - return CombineHashes(Hash(a), Hash(b)); - } + template <typename T, typename U> + size_t HashValues(const T& a, const U& b) { + return CombineHashes(Hash(a), Hash(b)); + } } -} +} diff --git a/library/cpp/messagebus/key_value_printer.cpp b/library/cpp/messagebus/key_value_printer.cpp index c8592145c7..75a82a10e2 100644 --- a/library/cpp/messagebus/key_value_printer.cpp +++ b/library/cpp/messagebus/key_value_printer.cpp @@ -4,11 +4,11 @@ TKeyValuePrinter::TKeyValuePrinter(const TString& sep) : Sep(sep) -{ -} +{ +} -TKeyValuePrinter::~TKeyValuePrinter() { -} +TKeyValuePrinter::~TKeyValuePrinter() { +} void TKeyValuePrinter::AddRowImpl(const TString& key, const TString& value, bool alignLeft) { Keys.push_back(key); diff --git a/library/cpp/messagebus/key_value_printer.h b/library/cpp/messagebus/key_value_printer.h index bca1fde50e..b0277b15b6 100644 --- a/library/cpp/messagebus/key_value_printer.h +++ b/library/cpp/messagebus/key_value_printer.h @@ -11,7 +11,7 @@ private: TVector<TString> Keys; TVector<TString> Values; TVector<bool> AlignLefts; - + public: TKeyValuePrinter(const TString& sep = TString(": ")); ~TKeyValuePrinter(); diff --git a/library/cpp/messagebus/latch.h b/library/cpp/messagebus/latch.h index 373f4c0e13..b6f64c2db3 100644 --- a/library/cpp/messagebus/latch.h +++ b/library/cpp/messagebus/latch.h @@ -9,12 +9,12 @@ private: TAtomic Locked; TMutex Mutex; TCondVar CondVar; - + public: - TLatch() - : Locked(0) - { - } + TLatch() + : Locked(0) + { + } void Wait() { // optimistic path diff --git a/library/cpp/messagebus/left_right_buffer.h b/library/cpp/messagebus/left_right_buffer.h index f937cefad0..a221324c23 100644 --- a/library/cpp/messagebus/left_right_buffer.h +++ b/library/cpp/messagebus/left_right_buffer.h @@ -4,75 +4,75 @@ #include <util/generic/noncopyable.h> #include <util/system/yassert.h> -namespace NBus { - namespace NPrivate { - class TLeftRightBuffer : TNonCopyable { - private: - TBuffer Buffer; - size_t Left; - - void CheckInvariant() { - Y_ASSERT(Left <= Buffer.Size()); - } - - public: - TLeftRightBuffer() - : Left(0) - { - } - - TBuffer& GetBuffer() { - return Buffer; - } - - size_t Capacity() { - return Buffer.Capacity(); - } - - void Clear() { - Buffer.Clear(); - Left = 0; - } - - void Reset() { - Buffer.Reset(); - Left = 0; - } - - void Compact() { - Buffer.ChopHead(Left); - Left = 0; - } - - char* LeftPos() { - return Buffer.Data() + Left; - } - - size_t LeftSize() { - return Left; - } - - void LeftProceed(size_t count) { - Y_ASSERT(count <= Size()); - Left += count; - } - - size_t Size() { - return Buffer.Size() - Left; - } - - bool Empty() { - return Size() == 0; - } - - char* RightPos() { - return Buffer.Data() + Buffer.Size(); - } - - size_t Avail() { - return Buffer.Avail(); - } - }; +namespace NBus { + namespace NPrivate { + class TLeftRightBuffer : TNonCopyable { + private: + TBuffer Buffer; + size_t Left; + + void CheckInvariant() { + Y_ASSERT(Left <= Buffer.Size()); + } + + public: + TLeftRightBuffer() + : Left(0) + { + } + + TBuffer& GetBuffer() { + return Buffer; + } + + size_t Capacity() { + return Buffer.Capacity(); + } + + void Clear() { + Buffer.Clear(); + Left = 0; + } + + void Reset() { + Buffer.Reset(); + Left = 0; + } + + void Compact() { + Buffer.ChopHead(Left); + Left = 0; + } + + char* LeftPos() { + return Buffer.Data() + Left; + } + + size_t LeftSize() { + return Left; + } + + void LeftProceed(size_t count) { + Y_ASSERT(count <= Size()); + Left += count; + } + + size_t Size() { + return Buffer.Size() - Left; + } + + bool Empty() { + return Size() == 0; + } + + char* RightPos() { + return Buffer.Data() + Buffer.Size(); + } + + size_t Avail() { + return Buffer.Avail(); + } + }; } -} +} diff --git a/library/cpp/messagebus/lfqueue_batch.h b/library/cpp/messagebus/lfqueue_batch.h index 8128d3154d..6c28f0ade6 100644 --- a/library/cpp/messagebus/lfqueue_batch.h +++ b/library/cpp/messagebus/lfqueue_batch.h @@ -5,17 +5,17 @@ #include <util/generic/vector.h> #include <util/thread/lfstack.h> -template <typename T, template <typename, class> class TVectorType = TVector> +template <typename T, template <typename, class> class TVectorType = TVector> class TLockFreeQueueBatch { private: TLockFreeStack<TVectorType<T, std::allocator<T>>*> Stack; - + public: bool IsEmpty() { return Stack.IsEmpty(); } - void EnqueueAll(TAutoPtr<TVectorType<T, std::allocator<T>>> vec) { + void EnqueueAll(TAutoPtr<TVectorType<T, std::allocator<T>>> vec) { Stack.Enqueue(vec.Release()); } @@ -24,7 +24,7 @@ public: Stack.DequeueAllSingleConsumer(vs.GetVector()); for (typename TVector<TVectorType<T, std::allocator<T>>*>::reverse_iterator i = vs.GetVector()->rbegin(); - i != vs.GetVector()->rend(); ++i) { + i != vs.GetVector()->rend(); ++i) { if (i == vs.GetVector()->rend()) { r->swap(**i); } else { diff --git a/library/cpp/messagebus/lfqueue_batch_ut.cpp b/library/cpp/messagebus/lfqueue_batch_ut.cpp index f80434c0d4..b60586d141 100644 --- a/library/cpp/messagebus/lfqueue_batch_ut.cpp +++ b/library/cpp/messagebus/lfqueue_batch_ut.cpp @@ -6,7 +6,7 @@ Y_UNIT_TEST_SUITE(TLockFreeQueueBatch) { Y_UNIT_TEST(Order1) { TLockFreeQueueBatch<unsigned> q; { - TAutoPtr<TVector<unsigned>> v(new TVector<unsigned>); + TAutoPtr<TVector<unsigned>> v(new TVector<unsigned>); v->push_back(0); v->push_back(1); q.EnqueueAll(v); @@ -28,13 +28,13 @@ Y_UNIT_TEST_SUITE(TLockFreeQueueBatch) { Y_UNIT_TEST(Order2) { TLockFreeQueueBatch<unsigned> q; { - TAutoPtr<TVector<unsigned>> v(new TVector<unsigned>); + TAutoPtr<TVector<unsigned>> v(new TVector<unsigned>); v->push_back(0); v->push_back(1); q.EnqueueAll(v); } { - TAutoPtr<TVector<unsigned>> v(new TVector<unsigned>); + TAutoPtr<TVector<unsigned>> v(new TVector<unsigned>); v->push_back(2); v->push_back(3); v->push_back(4); diff --git a/library/cpp/messagebus/local_flags.cpp b/library/cpp/messagebus/local_flags.cpp index 877e533f76..41602f00b9 100644 --- a/library/cpp/messagebus/local_flags.cpp +++ b/library/cpp/messagebus/local_flags.cpp @@ -14,15 +14,15 @@ TString NBus::NPrivate::LocalFlagSetToString(ui32 flags0) { ui32 flags = flags0; TStringStream ss; -#define P(name, value, ...) \ - do \ - if (flags & value) { \ - if (!ss.Str().empty()) { \ - ss << "|"; \ - } \ - ss << #name; \ - flags &= ~name; \ - } \ +#define P(name, value, ...) \ + do \ + if (flags & value) { \ + if (!ss.Str().empty()) { \ + ss << "|"; \ + } \ + ss << #name; \ + flags &= ~name; \ + } \ while (false); MESSAGE_LOCAL_FLAGS_MAP(P) if (flags != 0) { diff --git a/library/cpp/messagebus/local_flags.h b/library/cpp/messagebus/local_flags.h index f589283188..2677e6616d 100644 --- a/library/cpp/messagebus/local_flags.h +++ b/library/cpp/messagebus/local_flags.h @@ -5,22 +5,22 @@ #include <util/generic/string.h> #include <util/stream/output.h> -namespace NBus { - namespace NPrivate { -#define MESSAGE_LOCAL_FLAGS_MAP(XX) \ - XX(MESSAGE_REPLY_INTERNAL, 0x0001) \ - XX(MESSAGE_IN_WORK, 0x0002) \ - XX(MESSAGE_IN_FLIGHT_ON_CLIENT, 0x0004) \ - XX(MESSAGE_REPLY_IS_BEGING_SENT, 0x0008) \ - XX(MESSAGE_ONE_WAY_INTERNAL, 0x0010) \ - /**/ +namespace NBus { + namespace NPrivate { +#define MESSAGE_LOCAL_FLAGS_MAP(XX) \ + XX(MESSAGE_REPLY_INTERNAL, 0x0001) \ + XX(MESSAGE_IN_WORK, 0x0002) \ + XX(MESSAGE_IN_FLIGHT_ON_CLIENT, 0x0004) \ + XX(MESSAGE_REPLY_IS_BEGING_SENT, 0x0008) \ + XX(MESSAGE_ONE_WAY_INTERNAL, 0x0010) \ + /**/ - enum EMessageLocalFlags { - MESSAGE_LOCAL_FLAGS_MAP(ENUM_VALUE_GEN) - }; + enum EMessageLocalFlags { + MESSAGE_LOCAL_FLAGS_MAP(ENUM_VALUE_GEN) + }; - ENUM_TO_STRING(EMessageLocalFlags, MESSAGE_LOCAL_FLAGS_MAP) + ENUM_TO_STRING(EMessageLocalFlags, MESSAGE_LOCAL_FLAGS_MAP) - TString LocalFlagSetToString(ui32); - } -} + TString LocalFlagSetToString(ui32); + } +} diff --git a/library/cpp/messagebus/local_flags_ut.cpp b/library/cpp/messagebus/local_flags_ut.cpp index 189d73eb0f..361333892e 100644 --- a/library/cpp/messagebus/local_flags_ut.cpp +++ b/library/cpp/messagebus/local_flags_ut.cpp @@ -9,10 +9,10 @@ Y_UNIT_TEST_SUITE(EMessageLocalFlags) { Y_UNIT_TEST(TestLocalFlagSetToString) { UNIT_ASSERT_VALUES_EQUAL("0", LocalFlagSetToString(0)); UNIT_ASSERT_VALUES_EQUAL("MESSAGE_REPLY_INTERNAL", - LocalFlagSetToString(MESSAGE_REPLY_INTERNAL)); + LocalFlagSetToString(MESSAGE_REPLY_INTERNAL)); UNIT_ASSERT_VALUES_EQUAL("MESSAGE_IN_WORK|MESSAGE_IN_FLIGHT_ON_CLIENT", - LocalFlagSetToString(MESSAGE_IN_WORK | MESSAGE_IN_FLIGHT_ON_CLIENT)); + LocalFlagSetToString(MESSAGE_IN_WORK | MESSAGE_IN_FLIGHT_ON_CLIENT)); UNIT_ASSERT_VALUES_EQUAL("0xff3456", - LocalFlagSetToString(0xff3456)); + LocalFlagSetToString(0xff3456)); } } diff --git a/library/cpp/messagebus/local_tasks.h b/library/cpp/messagebus/local_tasks.h index d8e801a457..f7de96590d 100644 --- a/library/cpp/messagebus/local_tasks.h +++ b/library/cpp/messagebus/local_tasks.h @@ -9,8 +9,8 @@ private: public: TLocalTasks() : GotTasks(0) - { - } + { + } void AddTask() { AtomicSet(GotTasks, 1); diff --git a/library/cpp/messagebus/locator.cpp b/library/cpp/messagebus/locator.cpp index e38a35c426..f0f8da7cb0 100644 --- a/library/cpp/messagebus/locator.cpp +++ b/library/cpp/messagebus/locator.cpp @@ -1,5 +1,5 @@ //////////////////////////////////////////////////////////////////////////// -/// \file +/// \file /// \brief Implementation of locator service #include "locator.h" @@ -10,418 +10,418 @@ #include <util/system/hostname.h> namespace NBus { - using namespace NAddr; - - static TIpPort GetAddrPort(const IRemoteAddr& addr) { - switch (addr.Addr()->sa_family) { - case AF_INET: { - return ntohs(((const sockaddr_in*)addr.Addr())->sin_port); - } - - case AF_INET6: { - return ntohs(((const sockaddr_in6*)addr.Addr())->sin6_port); - } - - default: { - ythrow yexception() << "not implemented"; - break; - } + using namespace NAddr; + + static TIpPort GetAddrPort(const IRemoteAddr& addr) { + switch (addr.Addr()->sa_family) { + case AF_INET: { + return ntohs(((const sockaddr_in*)addr.Addr())->sin_port); + } + + case AF_INET6: { + return ntohs(((const sockaddr_in6*)addr.Addr())->sin6_port); + } + + default: { + ythrow yexception() << "not implemented"; + break; + } } - } - - static inline bool GetIp6AddressFromVector(const TVector<TNetAddr>& addrs, TNetAddr* addr) { - for (size_t i = 1; i < addrs.size(); ++i) { - if (addrs[i - 1].Addr()->sa_family == addrs[i].Addr()->sa_family) { - return false; - } - - if (GetAddrPort(addrs[i - 1]) != GetAddrPort(addrs[i])) { - return false; - } + } + + static inline bool GetIp6AddressFromVector(const TVector<TNetAddr>& addrs, TNetAddr* addr) { + for (size_t i = 1; i < addrs.size(); ++i) { + if (addrs[i - 1].Addr()->sa_family == addrs[i].Addr()->sa_family) { + return false; + } + + if (GetAddrPort(addrs[i - 1]) != GetAddrPort(addrs[i])) { + return false; + } } - for (size_t i = 0; i < addrs.size(); ++i) { - if (addrs[i].Addr()->sa_family == AF_INET6) { - *addr = addrs[i]; - return true; - } + for (size_t i = 0; i < addrs.size(); ++i) { + if (addrs[i].Addr()->sa_family == AF_INET6) { + *addr = addrs[i]; + return true; + } } - - return false; + + return false; } - EMessageStatus TBusProtocol::GetDestination(const TBusClientSession*, TBusMessage* mess, TBusLocator* locator, TNetAddr* addr) { - TBusService service = GetService(); - TBusKey key = GetKey(mess); - TVector<TNetAddr> addrs; - - /// check for special local key - if (key == YBUS_KEYLOCAL) { - locator->GetLocalAddresses(service, addrs); - } else { - /// lookup address/port in the locator table - locator->LocateAll(service, key, addrs); + EMessageStatus TBusProtocol::GetDestination(const TBusClientSession*, TBusMessage* mess, TBusLocator* locator, TNetAddr* addr) { + TBusService service = GetService(); + TBusKey key = GetKey(mess); + TVector<TNetAddr> addrs; + + /// check for special local key + if (key == YBUS_KEYLOCAL) { + locator->GetLocalAddresses(service, addrs); + } else { + /// lookup address/port in the locator table + locator->LocateAll(service, key, addrs); } - if (addrs.size() == 0) { - return MESSAGE_SERVICE_UNKNOWN; - } else if (addrs.size() == 1) { - *addr = addrs[0]; - } else { - if (!GetIp6AddressFromVector(addrs, addr)) { - /// default policy can't make choice for you here, overide GetDestination() function - /// to implement custom routing strategy for your service. - return MESSAGE_SERVICE_TOOMANY; - } + if (addrs.size() == 0) { + return MESSAGE_SERVICE_UNKNOWN; + } else if (addrs.size() == 1) { + *addr = addrs[0]; + } else { + if (!GetIp6AddressFromVector(addrs, addr)) { + /// default policy can't make choice for you here, overide GetDestination() function + /// to implement custom routing strategy for your service. + return MESSAGE_SERVICE_TOOMANY; + } } - - return MESSAGE_OK; + + return MESSAGE_OK; } - static const sockaddr_in* SockAddrIpV4(const IRemoteAddr& a) { - return (const sockaddr_in*)a.Addr(); + static const sockaddr_in* SockAddrIpV4(const IRemoteAddr& a) { + return (const sockaddr_in*)a.Addr(); } - static const sockaddr_in6* SockAddrIpV6(const IRemoteAddr& a) { - return (const sockaddr_in6*)a.Addr(); + static const sockaddr_in6* SockAddrIpV6(const IRemoteAddr& a) { + return (const sockaddr_in6*)a.Addr(); } - static bool IsAddressEqual(const IRemoteAddr& a1, const IRemoteAddr& a2) { - if (a1.Addr()->sa_family == a2.Addr()->sa_family) { - if (a1.Addr()->sa_family == AF_INET) { - return memcmp(&SockAddrIpV4(a1)->sin_addr, &SockAddrIpV4(a2)->sin_addr, sizeof(in_addr)) == 0; - } else { - return memcmp(&SockAddrIpV6(a1)->sin6_addr, &SockAddrIpV6(a2)->sin6_addr, sizeof(in6_addr)) == 0; - } + static bool IsAddressEqual(const IRemoteAddr& a1, const IRemoteAddr& a2) { + if (a1.Addr()->sa_family == a2.Addr()->sa_family) { + if (a1.Addr()->sa_family == AF_INET) { + return memcmp(&SockAddrIpV4(a1)->sin_addr, &SockAddrIpV4(a2)->sin_addr, sizeof(in_addr)) == 0; + } else { + return memcmp(&SockAddrIpV6(a1)->sin6_addr, &SockAddrIpV6(a2)->sin6_addr, sizeof(in6_addr)) == 0; + } } - return false; + return false; } - TBusLocator::TBusLocator() - : MyInterfaces(GetNetworkInterfaces()) - { - } + TBusLocator::TBusLocator() + : MyInterfaces(GetNetworkInterfaces()) + { + } - bool TBusLocator::TItem::operator<(const TItem& y) const { - const TItem& x = *this; + bool TBusLocator::TItem::operator<(const TItem& y) const { + const TItem& x = *this; - if (x.ServiceId == y.ServiceId) { - return (x.End < y.End) || ((x.End == y.End) && CompareByHost(x.Addr, y.Addr) < 0); + if (x.ServiceId == y.ServiceId) { + return (x.End < y.End) || ((x.End == y.End) && CompareByHost(x.Addr, y.Addr) < 0); } - return x.ServiceId < y.ServiceId; + return x.ServiceId < y.ServiceId; } - bool TBusLocator::TItem::operator==(const TItem& y) const { - return ServiceId == y.ServiceId && Start == y.Start && End == y.End && Addr == y.Addr; - } + bool TBusLocator::TItem::operator==(const TItem& y) const { + return ServiceId == y.ServiceId && Start == y.Start && End == y.End && Addr == y.Addr; + } - TBusLocator::TItem::TItem(TServiceId serviceId, TBusKey start, TBusKey end, const TNetAddr& addr) - : ServiceId(serviceId) - , Start(start) - , End(end) - , Addr(addr) - { + TBusLocator::TItem::TItem(TServiceId serviceId, TBusKey start, TBusKey end, const TNetAddr& addr) + : ServiceId(serviceId) + , Start(start) + , End(end) + , Addr(addr) + { } - bool TBusLocator::IsLocal(const TNetAddr& addr) { - for (const auto& myInterface : MyInterfaces) { - if (IsAddressEqual(addr, *myInterface.Address)) { - return true; - } - } + bool TBusLocator::IsLocal(const TNetAddr& addr) { + for (const auto& myInterface : MyInterfaces) { + if (IsAddressEqual(addr, *myInterface.Address)) { + return true; + } + } - return false; - } + return false; + } - TBusLocator::TServiceId TBusLocator::GetServiceId(const char* name) { - const char* c = ServiceIdSet.insert(name).first->c_str(); - return (ui64)c; + TBusLocator::TServiceId TBusLocator::GetServiceId(const char* name) { + const char* c = ServiceIdSet.insert(name).first->c_str(); + return (ui64)c; } - int TBusLocator::RegisterBreak(TBusService service, const TVector<TBusKey>& starts, const TNetAddr& addr) { - TGuard<TMutex> G(Lock); + int TBusLocator::RegisterBreak(TBusService service, const TVector<TBusKey>& starts, const TNetAddr& addr) { + TGuard<TMutex> G(Lock); - TServiceId serviceId = GetServiceId(service); - for (size_t i = 0; i < starts.size(); ++i) { - RegisterBreak(serviceId, starts[i], addr); - } - return 0; + TServiceId serviceId = GetServiceId(service); + for (size_t i = 0; i < starts.size(); ++i) { + RegisterBreak(serviceId, starts[i], addr); + } + return 0; } - int TBusLocator::RegisterBreak(TServiceId serviceId, const TBusKey start, const TNetAddr& addr) { - TItems::const_iterator it = Items.lower_bound(TItem(serviceId, 0, start, addr)); - TItems::const_iterator service_it = + int TBusLocator::RegisterBreak(TServiceId serviceId, const TBusKey start, const TNetAddr& addr) { + TItems::const_iterator it = Items.lower_bound(TItem(serviceId, 0, start, addr)); + TItems::const_iterator service_it = Items.lower_bound(TItem(serviceId, 0, 0, TNetAddr())); - THolder<TItem> left; - THolder<TItem> right; - if ((it != Items.end() || Items.begin() != Items.end()) && service_it != Items.end() && service_it->ServiceId == serviceId) { - if (it == Items.end()) { - --it; - } - const TItem& item = *it; - left.Reset(new TItem(serviceId, item.Start, - Max<TBusKey>(item.Start, start - 1), item.Addr)); - right.Reset(new TItem(serviceId, start, item.End, addr)); - Items.erase(*it); - } else { - left.Reset(new TItem(serviceId, YBUS_KEYMIN, start, addr)); - if (start < YBUS_KEYMAX) { - right.Reset(new TItem(serviceId, start + 1, YBUS_KEYMAX, addr)); - } + THolder<TItem> left; + THolder<TItem> right; + if ((it != Items.end() || Items.begin() != Items.end()) && service_it != Items.end() && service_it->ServiceId == serviceId) { + if (it == Items.end()) { + --it; + } + const TItem& item = *it; + left.Reset(new TItem(serviceId, item.Start, + Max<TBusKey>(item.Start, start - 1), item.Addr)); + right.Reset(new TItem(serviceId, start, item.End, addr)); + Items.erase(*it); + } else { + left.Reset(new TItem(serviceId, YBUS_KEYMIN, start, addr)); + if (start < YBUS_KEYMAX) { + right.Reset(new TItem(serviceId, start + 1, YBUS_KEYMAX, addr)); + } } - Items.insert(*left); - Items.insert(*right); - NormalizeBreaks(serviceId); - return 0; - } - - int TBusLocator::UnregisterBreak(TBusService service, const TNetAddr& addr) { - TGuard<TMutex> G(Lock); - - TServiceId serviceId = GetServiceId(service); - return UnregisterBreak(serviceId, addr); + Items.insert(*left); + Items.insert(*right); + NormalizeBreaks(serviceId); + return 0; } - int TBusLocator::UnregisterBreak(TServiceId serviceId, const TNetAddr& addr) { - int deleted = 0; - TItems::iterator it = Items.begin(); - while (it != Items.end()) { - const TItem& item = *it; - if (item.ServiceId != serviceId) { - ++it; - continue; - } - TItems::iterator itErase = it++; - if (item.ServiceId == serviceId && item.Addr == addr) { - Items.erase(itErase); - deleted += 1; - } + int TBusLocator::UnregisterBreak(TBusService service, const TNetAddr& addr) { + TGuard<TMutex> G(Lock); + + TServiceId serviceId = GetServiceId(service); + return UnregisterBreak(serviceId, addr); + } + + int TBusLocator::UnregisterBreak(TServiceId serviceId, const TNetAddr& addr) { + int deleted = 0; + TItems::iterator it = Items.begin(); + while (it != Items.end()) { + const TItem& item = *it; + if (item.ServiceId != serviceId) { + ++it; + continue; + } + TItems::iterator itErase = it++; + if (item.ServiceId == serviceId && item.Addr == addr) { + Items.erase(itErase); + deleted += 1; + } + } + + if (Items.begin() == Items.end()) { + return deleted; + } + TBusKey keyItem = YBUS_KEYMAX; + it = Items.end(); + TItems::iterator first = it; + do { + --it; + // item.Start is not used in set comparison function + // so you can't violate set sort order by changing it + // hence const_cast() + TItem& item = const_cast<TItem&>(*it); + if (item.ServiceId != serviceId) { + continue; + } + first = it; + if (item.End < keyItem) { + item.End = keyItem; + } + keyItem = item.Start - 1; + } while (it != Items.begin()); + + if (first != Items.end() && first->Start != 0) { + TItem item(serviceId, YBUS_KEYMIN, first->Start - 1, first->Addr); + Items.insert(item); } - if (Items.begin() == Items.end()) { - return deleted; - } - TBusKey keyItem = YBUS_KEYMAX; - it = Items.end(); - TItems::iterator first = it; - do { - --it; - // item.Start is not used in set comparison function - // so you can't violate set sort order by changing it - // hence const_cast() - TItem& item = const_cast<TItem&>(*it); - if (item.ServiceId != serviceId) { - continue; - } - first = it; - if (item.End < keyItem) { - item.End = keyItem; - } - keyItem = item.Start - 1; - } while (it != Items.begin()); - - if (first != Items.end() && first->Start != 0) { - TItem item(serviceId, YBUS_KEYMIN, first->Start - 1, first->Addr); - Items.insert(item); - } - - NormalizeBreaks(serviceId); + NormalizeBreaks(serviceId); return deleted; } - void TBusLocator::NormalizeBreaks(TServiceId serviceId) { - TItems::const_iterator first = Items.lower_bound(TItem(serviceId, YBUS_KEYMIN, YBUS_KEYMIN, TNetAddr())); - TItems::const_iterator last = Items.end(); + void TBusLocator::NormalizeBreaks(TServiceId serviceId) { + TItems::const_iterator first = Items.lower_bound(TItem(serviceId, YBUS_KEYMIN, YBUS_KEYMIN, TNetAddr())); + TItems::const_iterator last = Items.end(); - if ((Items.end() != first) && (first->ServiceId == serviceId)) { - if (serviceId != Max<TServiceId>()) { - last = Items.lower_bound(TItem(serviceId + 1, YBUS_KEYMIN, YBUS_KEYMIN, TNetAddr())); - } + if ((Items.end() != first) && (first->ServiceId == serviceId)) { + if (serviceId != Max<TServiceId>()) { + last = Items.lower_bound(TItem(serviceId + 1, YBUS_KEYMIN, YBUS_KEYMIN, TNetAddr())); + } - --last; - Y_ASSERT(Items.end() != last); - Y_ASSERT(last->ServiceId == serviceId); + --last; + Y_ASSERT(Items.end() != last); + Y_ASSERT(last->ServiceId == serviceId); - TItem& beg = const_cast<TItem&>(*first); - beg.Addr = last->Addr; + TItem& beg = const_cast<TItem&>(*first); + beg.Addr = last->Addr; } } - int TBusLocator::LocateAll(TBusService service, TBusKey key, TVector<TNetAddr>& addrs) { - TGuard<TMutex> G(Lock); - Y_VERIFY(addrs.empty(), "Non emtpy addresses"); + int TBusLocator::LocateAll(TBusService service, TBusKey key, TVector<TNetAddr>& addrs) { + TGuard<TMutex> G(Lock); + Y_VERIFY(addrs.empty(), "Non emtpy addresses"); - TServiceId serviceId = GetServiceId(service); - TItems::const_iterator it; + TServiceId serviceId = GetServiceId(service); + TItems::const_iterator it; - for (it = Items.lower_bound(TItem(serviceId, 0, key, TNetAddr())); - it != Items.end() && it->ServiceId == serviceId && it->Start <= key && key <= it->End; - ++it) { - const TItem& item = *it; - addrs.push_back(item.Addr); - } + for (it = Items.lower_bound(TItem(serviceId, 0, key, TNetAddr())); + it != Items.end() && it->ServiceId == serviceId && it->Start <= key && key <= it->End; + ++it) { + const TItem& item = *it; + addrs.push_back(item.Addr); + } - if (addrs.size() == 0) { - return -1; - } - return (int)addrs.size(); + if (addrs.size() == 0) { + return -1; + } + return (int)addrs.size(); } + + int TBusLocator::Locate(TBusService service, TBusKey key, TNetAddr* addr) { + TGuard<TMutex> G(Lock); - int TBusLocator::Locate(TBusService service, TBusKey key, TNetAddr* addr) { - TGuard<TMutex> G(Lock); + TServiceId serviceId = GetServiceId(service); + TItems::const_iterator it; - TServiceId serviceId = GetServiceId(service); - TItems::const_iterator it; + it = Items.lower_bound(TItem(serviceId, 0, key, TNetAddr())); - it = Items.lower_bound(TItem(serviceId, 0, key, TNetAddr())); + if (it != Items.end()) { + const TItem& item = *it; + if (item.ServiceId == serviceId && item.Start <= key && key < item.End) { + *addr = item.Addr; - if (it != Items.end()) { - const TItem& item = *it; - if (item.ServiceId == serviceId && item.Start <= key && key < item.End) { - *addr = item.Addr; + return 0; + } + } + + return -1; + } - return 0; - } + int TBusLocator::GetLocalPort(TBusService service) { + TGuard<TMutex> G(Lock); + TServiceId serviceId = GetServiceId(service); + TItems::const_iterator it; + int port = 0; + + for (it = Items.lower_bound(TItem(serviceId, 0, 0, TNetAddr())); it != Items.end(); ++it) { + const TItem& item = *it; + if (item.ServiceId != serviceId) { + break; + } + + if (IsLocal(item.Addr)) { + if (port != 0 && port != GetAddrPort(item.Addr)) { + Y_ASSERT(0 && "Can't decide which port to use."); + return 0; + } + port = GetAddrPort(item.Addr); + } } - return -1; + return port; } - int TBusLocator::GetLocalPort(TBusService service) { - TGuard<TMutex> G(Lock); - TServiceId serviceId = GetServiceId(service); - TItems::const_iterator it; - int port = 0; - - for (it = Items.lower_bound(TItem(serviceId, 0, 0, TNetAddr())); it != Items.end(); ++it) { - const TItem& item = *it; - if (item.ServiceId != serviceId) { - break; - } - - if (IsLocal(item.Addr)) { - if (port != 0 && port != GetAddrPort(item.Addr)) { - Y_ASSERT(0 && "Can't decide which port to use."); - return 0; - } - port = GetAddrPort(item.Addr); - } - } + int TBusLocator::GetLocalAddresses(TBusService service, TVector<TNetAddr>& addrs) { + TGuard<TMutex> G(Lock); + TServiceId serviceId = GetServiceId(service); + TItems::const_iterator it; - return port; - } + for (it = Items.lower_bound(TItem(serviceId, 0, 0, TNetAddr())); it != Items.end(); ++it) { + const TItem& item = *it; + if (item.ServiceId != serviceId) { + break; + } - int TBusLocator::GetLocalAddresses(TBusService service, TVector<TNetAddr>& addrs) { - TGuard<TMutex> G(Lock); - TServiceId serviceId = GetServiceId(service); - TItems::const_iterator it; - - for (it = Items.lower_bound(TItem(serviceId, 0, 0, TNetAddr())); it != Items.end(); ++it) { - const TItem& item = *it; - if (item.ServiceId != serviceId) { - break; - } - - if (IsLocal(item.Addr)) { - addrs.push_back(item.Addr); - } + if (IsLocal(item.Addr)) { + addrs.push_back(item.Addr); + } } - if (addrs.size() == 0) { - return -1; + if (addrs.size() == 0) { + return -1; } - return (int)addrs.size(); + return (int)addrs.size(); } - int TBusLocator::LocateHost(TBusService service, TBusKey key, TString* host, int* port, bool* isLocal) { - int ret; - TNetAddr addr; - ret = Locate(service, key, &addr); - if (ret != 0) { - return ret; - } - - { - TGuard<TMutex> G(Lock); - THostAddrMap::const_iterator it = HostAddrMap.find(addr); - if (it == HostAddrMap.end()) { - return -1; - } - *host = it->second; - } - - *port = GetAddrPort(addr); - if (isLocal != nullptr) { - *isLocal = IsLocal(addr); - } - return 0; + int TBusLocator::LocateHost(TBusService service, TBusKey key, TString* host, int* port, bool* isLocal) { + int ret; + TNetAddr addr; + ret = Locate(service, key, &addr); + if (ret != 0) { + return ret; + } + + { + TGuard<TMutex> G(Lock); + THostAddrMap::const_iterator it = HostAddrMap.find(addr); + if (it == HostAddrMap.end()) { + return -1; + } + *host = it->second; + } + + *port = GetAddrPort(addr); + if (isLocal != nullptr) { + *isLocal = IsLocal(addr); + } + return 0; } - int TBusLocator::LocateKeys(TBusService service, TBusKeyVec& keys, bool onlyLocal) { + int TBusLocator::LocateKeys(TBusService service, TBusKeyVec& keys, bool onlyLocal) { TGuard<TMutex> G(Lock); - Y_VERIFY(keys.empty(), "Non empty keys"); - - TServiceId serviceId = GetServiceId(service); - TItems::const_iterator it; - for (it = Items.begin(); it != Items.end(); ++it) { - const TItem& item = *it; - if (item.ServiceId != serviceId) { - continue; - } - if (onlyLocal && !IsLocal(item.Addr)) { - continue; - } - keys.push_back(std::make_pair(item.Start, item.End)); + Y_VERIFY(keys.empty(), "Non empty keys"); + + TServiceId serviceId = GetServiceId(service); + TItems::const_iterator it; + for (it = Items.begin(); it != Items.end(); ++it) { + const TItem& item = *it; + if (item.ServiceId != serviceId) { + continue; + } + if (onlyLocal && !IsLocal(item.Addr)) { + continue; + } + keys.push_back(std::make_pair(item.Start, item.End)); } - return (int)keys.size(); + return (int)keys.size(); } - int TBusLocator::Register(TBusService service, const char* hostName, int port, TBusKey start /*= YBUS_KEYMIN*/, TBusKey end /*= YBUS_KEYMAX*/, EIpVersion requireVersion /*= EIP_VERSION_4*/, EIpVersion preferVersion /*= EIP_VERSION_ANY*/) { - TNetAddr addr(hostName, port, requireVersion, preferVersion); // throws - { - TGuard<TMutex> G(Lock); - HostAddrMap[addr] = hostName; - } - Register(service, start, end, addr); - return 0; + int TBusLocator::Register(TBusService service, const char* hostName, int port, TBusKey start /*= YBUS_KEYMIN*/, TBusKey end /*= YBUS_KEYMAX*/, EIpVersion requireVersion /*= EIP_VERSION_4*/, EIpVersion preferVersion /*= EIP_VERSION_ANY*/) { + TNetAddr addr(hostName, port, requireVersion, preferVersion); // throws + { + TGuard<TMutex> G(Lock); + HostAddrMap[addr] = hostName; + } + Register(service, start, end, addr); + return 0; } - int TBusLocator::Register(TBusService service, TBusKey start, TBusKey end, const TNetworkAddress& na, EIpVersion requireVersion /*= EIP_VERSION_4*/, EIpVersion preferVersion /*= EIP_VERSION_ANY*/) { - TNetAddr addr(na, requireVersion, preferVersion); // throws - Register(service, start, end, addr); - return 0; + int TBusLocator::Register(TBusService service, TBusKey start, TBusKey end, const TNetworkAddress& na, EIpVersion requireVersion /*= EIP_VERSION_4*/, EIpVersion preferVersion /*= EIP_VERSION_ANY*/) { + TNetAddr addr(na, requireVersion, preferVersion); // throws + Register(service, start, end, addr); + return 0; } - int TBusLocator::Register(TBusService service, TBusKey start, TBusKey end, const TNetAddr& addr) { + int TBusLocator::Register(TBusService service, TBusKey start, TBusKey end, const TNetAddr& addr) { TGuard<TMutex> G(Lock); - TServiceId serviceId = GetServiceId(service); - TItems::const_iterator it; - - TItem itemToReg(serviceId, start, end, addr); - for (it = Items.lower_bound(TItem(serviceId, 0, start, TNetAddr())); - it != Items.end() && it->ServiceId == serviceId; - ++it) { - const TItem& item = *it; - if (item == itemToReg) { - return 0; - } - if ((item.Start < start && start < item.End) || (item.Start < end && end < item.End)) { - Y_FAIL("Overlap in registered keys with non-identical range"); - } - } - - Items.insert(itemToReg); - return 0; - } - - int TBusLocator::Unregister(TBusService service, TBusKey start, TBusKey end) { - TGuard<TMutex> G(Lock); - TServiceId serviceId = GetServiceId(service); - Items.erase(TItem(serviceId, start, end, TNetAddr())); - return 0; + TServiceId serviceId = GetServiceId(service); + TItems::const_iterator it; + + TItem itemToReg(serviceId, start, end, addr); + for (it = Items.lower_bound(TItem(serviceId, 0, start, TNetAddr())); + it != Items.end() && it->ServiceId == serviceId; + ++it) { + const TItem& item = *it; + if (item == itemToReg) { + return 0; + } + if ((item.Start < start && start < item.End) || (item.Start < end && end < item.End)) { + Y_FAIL("Overlap in registered keys with non-identical range"); + } + } + + Items.insert(itemToReg); + return 0; + } + + int TBusLocator::Unregister(TBusService service, TBusKey start, TBusKey end) { + TGuard<TMutex> G(Lock); + TServiceId serviceId = GetServiceId(service); + Items.erase(TItem(serviceId, start, end, TNetAddr())); + return 0; } } diff --git a/library/cpp/messagebus/locator.h b/library/cpp/messagebus/locator.h index f8556a3fce..2c455ac53e 100644 --- a/library/cpp/messagebus/locator.h +++ b/library/cpp/messagebus/locator.h @@ -10,84 +10,84 @@ #include <util/system/mutex.h> namespace NBus { - /////////////////////////////////////////////// - /// \brief Client interface to locator service + /////////////////////////////////////////////// + /// \brief Client interface to locator service - /// This interface abstracts clustering/location service that - /// allows clients find servers (address, port) using "name" and "key". - /// The instance lives in TBusMessageQueue-object, but can be shared by different queues. - class TBusLocator: public TAtomicRefCount<TBusLocator>, public TNonCopyable { - private: - typedef ui64 TServiceId; - typedef TSet<TString> TServiceIdSet; - TServiceIdSet ServiceIdSet; - TServiceId GetServiceId(const char* name); + /// This interface abstracts clustering/location service that + /// allows clients find servers (address, port) using "name" and "key". + /// The instance lives in TBusMessageQueue-object, but can be shared by different queues. + class TBusLocator: public TAtomicRefCount<TBusLocator>, public TNonCopyable { + private: + typedef ui64 TServiceId; + typedef TSet<TString> TServiceIdSet; + TServiceIdSet ServiceIdSet; + TServiceId GetServiceId(const char* name); - typedef TMap<TNetAddr, TString> THostAddrMap; - THostAddrMap HostAddrMap; + typedef TMap<TNetAddr, TString> THostAddrMap; + THostAddrMap HostAddrMap; - TNetworkInterfaceList MyInterfaces; + TNetworkInterfaceList MyInterfaces; - struct TItem { - TServiceId ServiceId; - TBusKey Start; - TBusKey End; - TNetAddr Addr; + struct TItem { + TServiceId ServiceId; + TBusKey Start; + TBusKey End; + TNetAddr Addr; - bool operator<(const TItem& y) const; + bool operator<(const TItem& y) const; - bool operator==(const TItem& y) const; + bool operator==(const TItem& y) const; - TItem(TServiceId serviceId, TBusKey start, TBusKey end, const TNetAddr& addr); - }; + TItem(TServiceId serviceId, TBusKey start, TBusKey end, const TNetAddr& addr); + }; - typedef TMultiSet<TItem> TItems; - TItems Items; - TMutex Lock; + typedef TMultiSet<TItem> TItems; + TItems Items; + TMutex Lock; - int RegisterBreak(TServiceId serviceId, const TBusKey start, const TNetAddr& addr); - int UnregisterBreak(TServiceId serviceId, const TNetAddr& addr); + int RegisterBreak(TServiceId serviceId, const TBusKey start, const TNetAddr& addr); + int UnregisterBreak(TServiceId serviceId, const TNetAddr& addr); - void NormalizeBreaks(TServiceId serviceId); + void NormalizeBreaks(TServiceId serviceId); - private: - int Register(TBusService service, TBusKey start, TBusKey end, const TNetAddr& addr); + private: + int Register(TBusService service, TBusKey start, TBusKey end, const TNetAddr& addr); - public: - /// creates instance that obtains location table from locator server (not implemented) - TBusLocator(); + public: + /// creates instance that obtains location table from locator server (not implemented) + TBusLocator(); - /// returns true if this address is on the same node for YBUS_KEYLOCAL - bool IsLocal(const TNetAddr& addr); + /// returns true if this address is on the same node for YBUS_KEYLOCAL + bool IsLocal(const TNetAddr& addr); - /// returns first address for service and key - int Locate(TBusService service, TBusKey key, TNetAddr* addr); + /// returns first address for service and key + int Locate(TBusService service, TBusKey key, TNetAddr* addr); - /// returns all addresses mathing service and key - int LocateAll(TBusService service, TBusKey key, TVector<TNetAddr>& addrs); + /// returns all addresses mathing service and key + int LocateAll(TBusService service, TBusKey key, TVector<TNetAddr>& addrs); - /// returns actual host name for service and key - int LocateHost(TBusService service, TBusKey key, TString* host, int* port, bool* isLocal = nullptr); + /// returns actual host name for service and key + int LocateHost(TBusService service, TBusKey key, TString* host, int* port, bool* isLocal = nullptr); - /// returns all key ranges for the given service - int LocateKeys(TBusService service, TBusKeyVec& keys, bool onlyLocal = false); + /// returns all key ranges for the given service + int LocateKeys(TBusService service, TBusKeyVec& keys, bool onlyLocal = false); - /// returns port on the local node for the service - int GetLocalPort(TBusService service); + /// returns port on the local node for the service + int GetLocalPort(TBusService service); - /// returns addresses of the local node for the service - int GetLocalAddresses(TBusService service, TVector<TNetAddr>& addrs); + /// returns addresses of the local node for the service + int GetLocalAddresses(TBusService service, TVector<TNetAddr>& addrs); - /// register service instance - int Register(TBusService service, TBusKey start, TBusKey end, const TNetworkAddress& addr, EIpVersion requireVersion = EIP_VERSION_4, EIpVersion preferVersion = EIP_VERSION_ANY); - /// @throws yexception - int Register(TBusService service, const char* host, int port, TBusKey start = YBUS_KEYMIN, TBusKey end = YBUS_KEYMAX, EIpVersion requireVersion = EIP_VERSION_4, EIpVersion preferVersion = EIP_VERSION_ANY); + /// register service instance + int Register(TBusService service, TBusKey start, TBusKey end, const TNetworkAddress& addr, EIpVersion requireVersion = EIP_VERSION_4, EIpVersion preferVersion = EIP_VERSION_ANY); + /// @throws yexception + int Register(TBusService service, const char* host, int port, TBusKey start = YBUS_KEYMIN, TBusKey end = YBUS_KEYMAX, EIpVersion requireVersion = EIP_VERSION_4, EIpVersion preferVersion = EIP_VERSION_ANY); - /// unregister service instance - int Unregister(TBusService service, TBusKey start, TBusKey end); + /// unregister service instance + int Unregister(TBusService service, TBusKey start, TBusKey end); - int RegisterBreak(TBusService service, const TVector<TBusKey>& starts, const TNetAddr& addr); - int UnregisterBreak(TBusService service, const TNetAddr& addr); - }; + int RegisterBreak(TBusService service, const TVector<TBusKey>& starts, const TNetAddr& addr); + int UnregisterBreak(TBusService service, const TNetAddr& addr); + }; -} +} diff --git a/library/cpp/messagebus/mb_lwtrace.h b/library/cpp/messagebus/mb_lwtrace.h index e62728b265..f8d5c7afe7 100644 --- a/library/cpp/messagebus/mb_lwtrace.h +++ b/library/cpp/messagebus/mb_lwtrace.h @@ -4,12 +4,12 @@ #include <util/generic/string.h> -#define LWTRACE_MESSAGEBUS_PROVIDER(PROBE, EVENT, GROUPS, TYPES, NAMES) \ +#define LWTRACE_MESSAGEBUS_PROVIDER(PROBE, EVENT, GROUPS, TYPES, NAMES) \ PROBE(Error, GROUPS("MessagebusRare"), TYPES(TString, TString, TString), NAMES("status", "address", "misc")) \ - PROBE(ServerUnknownVersion, GROUPS("MessagebusRare"), TYPES(TString, ui32), NAMES("address", "version")) \ - PROBE(Accepted, GROUPS("MessagebusRare"), TYPES(TString), NAMES("address")) \ - PROBE(Disconnected, GROUPS("MessagebusRare"), TYPES(TString), NAMES("address")) \ - PROBE(Read, GROUPS(), TYPES(ui32), NAMES("size")) \ + PROBE(ServerUnknownVersion, GROUPS("MessagebusRare"), TYPES(TString, ui32), NAMES("address", "version")) \ + PROBE(Accepted, GROUPS("MessagebusRare"), TYPES(TString), NAMES("address")) \ + PROBE(Disconnected, GROUPS("MessagebusRare"), TYPES(TString), NAMES("address")) \ + PROBE(Read, GROUPS(), TYPES(ui32), NAMES("size")) \ /**/ LWTRACE_DECLARE_PROVIDER(LWTRACE_MESSAGEBUS_PROVIDER) diff --git a/library/cpp/messagebus/memory.h b/library/cpp/messagebus/memory.h index b2c0544491..2c79d3ec7a 100644 --- a/library/cpp/messagebus/memory.h +++ b/library/cpp/messagebus/memory.h @@ -11,14 +11,14 @@ #define CACHE_LINE_PADDING char UNIQUE_NAME[CACHE_LINE_SIZE]; static inline void* MallocAligned(size_t size, size_t alignment) { - void** ptr = (void**)malloc(size + alignment + sizeof(size_t*)); + void** ptr = (void**)malloc(size + alignment + sizeof(size_t*)); if (!ptr) { return nullptr; } size_t mask = ~(alignment - 1); intptr_t roundedDown = intptr_t(ptr) & mask; - void** alignedPtr = (void**)(roundedDown + alignment); + void** alignedPtr = (void**)(roundedDown + alignment); alignedPtr[-1] = ptr; return alignedPtr; } @@ -28,8 +28,8 @@ static inline void FreeAligned(void* ptr) { return; } - void** typedPtr = (void**)ptr; - void* originalPtr = typedPtr[-1]; + void** typedPtr = (void**)ptr; + void* originalPtr = typedPtr[-1]; free(originalPtr); } diff --git a/library/cpp/messagebus/message.cpp b/library/cpp/messagebus/message.cpp index bfa7ed8e9b..a57e235977 100644 --- a/library/cpp/messagebus/message.cpp +++ b/library/cpp/messagebus/message.cpp @@ -10,185 +10,185 @@ using namespace NBus; namespace NBus { - using namespace NBus::NPrivate; - - TBusIdentity::TBusIdentity() - : MessageId(0) - , Size(0) - , Flags(0) - , LocalFlags(0) - { - } + using namespace NBus::NPrivate; + + TBusIdentity::TBusIdentity() + : MessageId(0) + , Size(0) + , Flags(0) + , LocalFlags(0) + { + } - TBusIdentity::~TBusIdentity() { + TBusIdentity::~TBusIdentity() { // TODO: print local flags #ifndef NDEBUG - Y_VERIFY(LocalFlags == 0, "local flags must be zero at this point; message type is %s", + Y_VERIFY(LocalFlags == 0, "local flags must be zero at this point; message type is %s", MessageType.value_or("unknown").c_str()); #else - Y_VERIFY(LocalFlags == 0, "local flags must be zero at this point"); + Y_VERIFY(LocalFlags == 0, "local flags must be zero at this point"); #endif - } - - TNetAddr TBusIdentity::GetNetAddr() const { - if (!!Connection) { - return Connection->GetAddr(); - } else { - Y_FAIL(); - } - } - - void TBusIdentity::Pack(char* dest) { - memcpy(dest, this, sizeof(TBusIdentity)); - LocalFlags = 0; - - // prevent decref - new (&Connection) TIntrusivePtr<TRemoteServerConnection>; - } - - void TBusIdentity::Unpack(const char* src) { - Y_VERIFY(LocalFlags == 0); - Y_VERIFY(!Connection); - - memcpy(this, src, sizeof(TBusIdentity)); - } - - void TBusHeader::GenerateId() { - for (;;) { - Id = RandomNumber<TBusKey>(); - // Skip reserved ids - if (IsBusKeyValid(Id)) - return; - } - } - - TBusMessage::TBusMessage(ui16 type, int approxsize) - //: TCtr("BusMessage") - : TRefCounted<TBusMessage, TAtomicCounter, TDelete>(1) - , LocalFlags(0) - , RequestSize(0) - , Data(nullptr) - { - Y_UNUSED(approxsize); - GetHeader()->Type = type; - DoReset(); - } - - TBusMessage::TBusMessage(ECreateUninitialized) - //: TCtr("BusMessage") - : TRefCounted<TBusMessage, TAtomicCounter, TDelete>(1) - , LocalFlags(0) - , Data(nullptr) - { - } - - TString TBusMessage::Describe() const { + } + + TNetAddr TBusIdentity::GetNetAddr() const { + if (!!Connection) { + return Connection->GetAddr(); + } else { + Y_FAIL(); + } + } + + void TBusIdentity::Pack(char* dest) { + memcpy(dest, this, sizeof(TBusIdentity)); + LocalFlags = 0; + + // prevent decref + new (&Connection) TIntrusivePtr<TRemoteServerConnection>; + } + + void TBusIdentity::Unpack(const char* src) { + Y_VERIFY(LocalFlags == 0); + Y_VERIFY(!Connection); + + memcpy(this, src, sizeof(TBusIdentity)); + } + + void TBusHeader::GenerateId() { + for (;;) { + Id = RandomNumber<TBusKey>(); + // Skip reserved ids + if (IsBusKeyValid(Id)) + return; + } + } + + TBusMessage::TBusMessage(ui16 type, int approxsize) + //: TCtr("BusMessage") + : TRefCounted<TBusMessage, TAtomicCounter, TDelete>(1) + , LocalFlags(0) + , RequestSize(0) + , Data(nullptr) + { + Y_UNUSED(approxsize); + GetHeader()->Type = type; + DoReset(); + } + + TBusMessage::TBusMessage(ECreateUninitialized) + //: TCtr("BusMessage") + : TRefCounted<TBusMessage, TAtomicCounter, TDelete>(1) + , LocalFlags(0) + , Data(nullptr) + { + } + + TString TBusMessage::Describe() const { return Sprintf("object type: %s, message type: %d", TypeName(*this).data(), int(GetHeader()->Type)); - } + } - TBusMessage::~TBusMessage() { + TBusMessage::~TBusMessage() { #ifndef NDEBUG - Y_VERIFY(GetHeader()->Id != YBUS_KEYINVALID, "must not be invalid key, message type: %d, ", int(Type)); - GetHeader()->Id = YBUS_KEYINVALID; - Data = (void*)17; - CheckClean(); + Y_VERIFY(GetHeader()->Id != YBUS_KEYINVALID, "must not be invalid key, message type: %d, ", int(Type)); + GetHeader()->Id = YBUS_KEYINVALID; + Data = (void*)17; + CheckClean(); #endif - } - - void TBusMessage::DoReset() { - GetHeader()->SendTime = 0; - GetHeader()->Size = 0; - GetHeader()->FlagsInternal = 0; - GetHeader()->GenerateId(); - GetHeader()->SetVersionInternal(); - } - - void TBusMessage::Reset() { - CheckClean(); - DoReset(); - } - - void TBusMessage::CheckClean() const { - if (Y_UNLIKELY(LocalFlags != 0)) { - TString describe = Describe(); - TString localFlags = LocalFlagSetToString(LocalFlags); + } + + void TBusMessage::DoReset() { + GetHeader()->SendTime = 0; + GetHeader()->Size = 0; + GetHeader()->FlagsInternal = 0; + GetHeader()->GenerateId(); + GetHeader()->SetVersionInternal(); + } + + void TBusMessage::Reset() { + CheckClean(); + DoReset(); + } + + void TBusMessage::CheckClean() const { + if (Y_UNLIKELY(LocalFlags != 0)) { + TString describe = Describe(); + TString localFlags = LocalFlagSetToString(LocalFlags); Y_FAIL("message local flags must be zero, got: %s, message: %s", localFlags.data(), describe.data()); - } - } - - /////////////////////////////////////////////////////// - /// \brief Unpacks header from network order - - /// \todo ntoh instead of memcpy - int TBusHeader::ReadHeader(TArrayRef<const char> data) { - Y_ASSERT(data.size() >= sizeof(TBusHeader)); - memcpy(this, data.data(), sizeof(TBusHeader)); - return sizeof(TBusHeader); - } - - /////////////////////////////////////////////////////// - /// \brief Packs header to network order - - ////////////////////////////////////////////////////////// - /// \brief serialize message identity to be used to construct reply message - - /// function stores messageid, flags and connection reply address into the buffer - /// that can later be used to construct a reply to the message - void TBusMessage::GetIdentity(TBusIdentity& data) const { - data.MessageId = GetHeader()->Id; - data.Size = GetHeader()->Size; - data.Flags = GetHeader()->FlagsInternal; - //data.LocalFlags = LocalFlags; - } - - //////////////////////////////////////////////////////////// - /// \brief set message identity from serialized form - - /// function restores messageid, flags and connection reply address from the buffer - /// into the reply message - void TBusMessage::SetIdentity(const TBusIdentity& data) { - // TODO: wrong assertion: YBUS_KEYMIN is 0 - Y_ASSERT(data.MessageId != 0); - bool compressed = IsCompressed(); - GetHeader()->Id = data.MessageId; - GetHeader()->FlagsInternal = data.Flags; - LocalFlags = data.LocalFlags & ~MESSAGE_IN_WORK; - ReplyTo = data.Connection->PeerAddrSocketAddr; - SetCompressed(compressed || IsCompressedResponse()); - } - - void TBusMessage::SetCompressed(bool v) { - if (v) { - GetHeader()->FlagsInternal |= MESSAGE_COMPRESS_INTERNAL; - } else { - GetHeader()->FlagsInternal &= ~(MESSAGE_COMPRESS_INTERNAL); - } - } - - void TBusMessage::SetCompressedResponse(bool v) { - if (v) { - GetHeader()->FlagsInternal |= MESSAGE_COMPRESS_RESPONSE; - } else { - GetHeader()->FlagsInternal &= ~(MESSAGE_COMPRESS_RESPONSE); - } - } - - TString TBusIdentity::ToString() const { - TStringStream ss; - ss << "msg-id=" << MessageId - << " size=" << Size; - if (!!Connection) { - ss << " conn=" << Connection->GetAddr(); - } - ss + } + } + + /////////////////////////////////////////////////////// + /// \brief Unpacks header from network order + + /// \todo ntoh instead of memcpy + int TBusHeader::ReadHeader(TArrayRef<const char> data) { + Y_ASSERT(data.size() >= sizeof(TBusHeader)); + memcpy(this, data.data(), sizeof(TBusHeader)); + return sizeof(TBusHeader); + } + + /////////////////////////////////////////////////////// + /// \brief Packs header to network order + + ////////////////////////////////////////////////////////// + /// \brief serialize message identity to be used to construct reply message + + /// function stores messageid, flags and connection reply address into the buffer + /// that can later be used to construct a reply to the message + void TBusMessage::GetIdentity(TBusIdentity& data) const { + data.MessageId = GetHeader()->Id; + data.Size = GetHeader()->Size; + data.Flags = GetHeader()->FlagsInternal; + //data.LocalFlags = LocalFlags; + } + + //////////////////////////////////////////////////////////// + /// \brief set message identity from serialized form + + /// function restores messageid, flags and connection reply address from the buffer + /// into the reply message + void TBusMessage::SetIdentity(const TBusIdentity& data) { + // TODO: wrong assertion: YBUS_KEYMIN is 0 + Y_ASSERT(data.MessageId != 0); + bool compressed = IsCompressed(); + GetHeader()->Id = data.MessageId; + GetHeader()->FlagsInternal = data.Flags; + LocalFlags = data.LocalFlags & ~MESSAGE_IN_WORK; + ReplyTo = data.Connection->PeerAddrSocketAddr; + SetCompressed(compressed || IsCompressedResponse()); + } + + void TBusMessage::SetCompressed(bool v) { + if (v) { + GetHeader()->FlagsInternal |= MESSAGE_COMPRESS_INTERNAL; + } else { + GetHeader()->FlagsInternal &= ~(MESSAGE_COMPRESS_INTERNAL); + } + } + + void TBusMessage::SetCompressedResponse(bool v) { + if (v) { + GetHeader()->FlagsInternal |= MESSAGE_COMPRESS_RESPONSE; + } else { + GetHeader()->FlagsInternal &= ~(MESSAGE_COMPRESS_RESPONSE); + } + } + + TString TBusIdentity::ToString() const { + TStringStream ss; + ss << "msg-id=" << MessageId + << " size=" << Size; + if (!!Connection) { + ss << " conn=" << Connection->GetAddr(); + } + ss << " flags=" << Flags << " local-flags=" << LocalFlags #ifndef NDEBUG << " msg-type= " << MessageType.value_or("unknown").c_str() #endif ; - return ss.Str(); - } + return ss.Str(); + } } diff --git a/library/cpp/messagebus/message.h b/library/cpp/messagebus/message.h index 005ca10c65..275f1859df 100644 --- a/library/cpp/messagebus/message.h +++ b/library/cpp/messagebus/message.h @@ -18,98 +18,98 @@ #include <typeinfo> namespace NBus { - /////////////////////////////////////////////////////////////////// - /// \brief Structure to preserve identity from message to reply - struct TBusIdentity : TNonCopyable { - friend class TBusMessage; - friend class NPrivate::TRemoteServerSession; - friend struct NPrivate::TClientRequestImpl; - friend class TOnMessageContext; - - // TODO: make private - TBusKey MessageId; - - private: - ui32 Size; - TIntrusivePtr<NPrivate::TRemoteServerConnection> Connection; - ui16 Flags; - ui32 LocalFlags; - TInstant RecvTime; + /////////////////////////////////////////////////////////////////// + /// \brief Structure to preserve identity from message to reply + struct TBusIdentity : TNonCopyable { + friend class TBusMessage; + friend class NPrivate::TRemoteServerSession; + friend struct NPrivate::TClientRequestImpl; + friend class TOnMessageContext; + + // TODO: make private + TBusKey MessageId; + + private: + ui32 Size; + TIntrusivePtr<NPrivate::TRemoteServerConnection> Connection; + ui16 Flags; + ui32 LocalFlags; + TInstant RecvTime; #ifndef NDEBUG std::optional<TString> MessageType; #endif private: - // TODO: drop - TNetAddr GetNetAddr() const; - - public: - void Pack(char* dest); - void Unpack(const char* src); - - bool IsInWork() const { - return LocalFlags & NPrivate::MESSAGE_IN_WORK; - } - - // for internal use only - void BeginWork() { - SetInWork(true); - } - - // for internal use only - void EndWork() { - SetInWork(false); - } - - TBusIdentity(); - ~TBusIdentity(); - - void Swap(TBusIdentity& that) { - DoSwap(MessageId, that.MessageId); - DoSwap(Size, that.Size); - DoSwap(Connection, that.Connection); - DoSwap(Flags, that.Flags); - DoSwap(LocalFlags, that.LocalFlags); - DoSwap(RecvTime, that.RecvTime); + // TODO: drop + TNetAddr GetNetAddr() const; + + public: + void Pack(char* dest); + void Unpack(const char* src); + + bool IsInWork() const { + return LocalFlags & NPrivate::MESSAGE_IN_WORK; + } + + // for internal use only + void BeginWork() { + SetInWork(true); + } + + // for internal use only + void EndWork() { + SetInWork(false); + } + + TBusIdentity(); + ~TBusIdentity(); + + void Swap(TBusIdentity& that) { + DoSwap(MessageId, that.MessageId); + DoSwap(Size, that.Size); + DoSwap(Connection, that.Connection); + DoSwap(Flags, that.Flags); + DoSwap(LocalFlags, that.LocalFlags); + DoSwap(RecvTime, that.RecvTime); #ifndef NDEBUG - DoSwap(MessageType, that.MessageType); + DoSwap(MessageType, that.MessageType); #endif - } - - TString ToString() const; - - private: - void SetInWork(bool inWork) { - if (LocalFlags == 0 && inWork) { - LocalFlags = NPrivate::MESSAGE_IN_WORK; - } else if (LocalFlags == NPrivate::MESSAGE_IN_WORK && !inWork) { - LocalFlags = 0; - } else { - Y_FAIL("impossible combination of flag and parameter: %s %d", - inWork ? "true" : "false", unsigned(LocalFlags)); - } + } + + TString ToString() const; + + private: + void SetInWork(bool inWork) { + if (LocalFlags == 0 && inWork) { + LocalFlags = NPrivate::MESSAGE_IN_WORK; + } else if (LocalFlags == NPrivate::MESSAGE_IN_WORK && !inWork) { + LocalFlags = 0; + } else { + Y_FAIL("impossible combination of flag and parameter: %s %d", + inWork ? "true" : "false", unsigned(LocalFlags)); + } } void SetMessageType(const std::type_info& messageTypeInfo) { #ifndef NDEBUG - Y_VERIFY(!MessageType, "state check"); + Y_VERIFY(!MessageType, "state check"); MessageType = TypeName(messageTypeInfo); #else Y_UNUSED(messageTypeInfo); #endif - } - }; + } + }; - static const size_t BUS_IDENTITY_PACKED_SIZE = sizeof(TBusIdentity); + static const size_t BUS_IDENTITY_PACKED_SIZE = sizeof(TBusIdentity); - /////////////////////////////////////////////////////////////// - /// \brief Message flags in TBusHeader.Flags - enum EMessageFlags { - MESSAGE_COMPRESS_INTERNAL = 0x8000, ///< message is compressed - MESSAGE_COMPRESS_RESPONSE = 0x4000, ///< message prefers compressed response - MESSAGE_VERSION_INTERNAL = 0x00F0, ///< these bits are used as version - }; + /////////////////////////////////////////////////////////////// + /// \brief Message flags in TBusHeader.Flags + enum EMessageFlags { + MESSAGE_COMPRESS_INTERNAL = 0x8000, ///< message is compressed + MESSAGE_COMPRESS_RESPONSE = 0x4000, ///< message prefers compressed response + MESSAGE_VERSION_INTERNAL = 0x00F0, ///< these bits are used as version + }; ////////////////////////////////////////////////////////// /// \brief Message header present in all message send and received @@ -117,8 +117,8 @@ namespace NBus { /// This header is send into the wire. /// \todo fix for low/high end, 32/64bit some day #pragma pack(1) - struct TBusHeader { - friend class TBusMessage; + struct TBusHeader { + friend class TBusMessage; TBusKey Id = 0; ///< unique message ID ui32 Size = 0; ///< total size of the message @@ -126,147 +126,147 @@ namespace NBus { ui16 FlagsInternal = 0; ///< TRACE is one of the flags ui16 Type = 0; ///< to be used by TBusProtocol - int GetVersionInternal() { - return (FlagsInternal & MESSAGE_VERSION_INTERNAL) >> 4; - } - void SetVersionInternal(unsigned ver = YBUS_VERSION) { - FlagsInternal |= (ver << 4); - } - - public: - TBusHeader() { - } - TBusHeader(TArrayRef<const char> data) { - ReadHeader(data); - } - - private: - /// function for serialization/deserialization of the header - /// returns number of bytes written/read - int ReadHeader(TArrayRef<const char> data); - - void GenerateId(); - }; + int GetVersionInternal() { + return (FlagsInternal & MESSAGE_VERSION_INTERNAL) >> 4; + } + void SetVersionInternal(unsigned ver = YBUS_VERSION) { + FlagsInternal |= (ver << 4); + } + + public: + TBusHeader() { + } + TBusHeader(TArrayRef<const char> data) { + ReadHeader(data); + } + + private: + /// function for serialization/deserialization of the header + /// returns number of bytes written/read + int ReadHeader(TArrayRef<const char> data); + + void GenerateId(); + }; #pragma pack() -#define TBUSMAX_MESSAGE 26 * 1024 * 1024 + sizeof(NBus::TBusHeader) ///< is't it enough? -#define TBUSMIN_MESSAGE sizeof(NBus::TBusHeader) ///< can't be less then header - - inline bool IsVersionNegotiation(const NBus::TBusHeader& header) { - return header.Id == 0 && header.Size == sizeof(TBusHeader); - } - - ////////////////////////////////////////////////////////// - /// \brief Base class for all messages passed in the system - - enum ECreateUninitialized { - MESSAGE_CREATE_UNINITIALIZED, - }; - - class TBusMessage - : protected TBusHeader, - public TRefCounted<TBusMessage, TAtomicCounter, TDelete>, - private TNonCopyable { - friend class TLocalSession; - friend struct ::NBus::NPrivate::TBusSessionImpl; - friend class ::NBus::NPrivate::TRemoteServerSession; - friend class ::NBus::NPrivate::TRemoteClientSession; - friend class ::NBus::NPrivate::TRemoteConnection; - friend class ::NBus::NPrivate::TRemoteClientConnection; - friend class ::NBus::NPrivate::TRemoteServerConnection; - friend struct ::NBus::NPrivate::TBusMessagePtrAndHeader; - - private: - ui32 LocalFlags; - - /// connection identity for reply set by PushMessage() - NPrivate::TBusSocketAddr ReplyTo; - // server-side response only, hack - ui32 RequestSize; - - TInstant RecvTime; - - public: - /// constructor to create messages on sending end - TBusMessage(ui16 type, int approxsize = sizeof(TBusHeader)); - - /// constructor with serialzed data to examine the header - TBusMessage(ECreateUninitialized); - - // slow, for diagnostics only - virtual TString Describe() const; - - // must be called if this message object needs to be reused - void Reset(); - - void CheckClean() const; - - void SetCompressed(bool); - void SetCompressedResponse(bool); - - private: - bool IsCompressed() const { - return FlagsInternal & MESSAGE_COMPRESS_INTERNAL; - } - bool IsCompressedResponse() const { - return FlagsInternal & MESSAGE_COMPRESS_RESPONSE; - } - - public: - /// can have private data to destroy - virtual ~TBusMessage(); - - /// returns header of the message - TBusHeader* GetHeader() { - return this; - } - const TBusHeader* GetHeader() const { - return this; - } - - /// helper to return type for protocol object to unpack object - static ui16 GetType(TArrayRef<const char> data) { - return TBusHeader(data).Type; - } - - /// returns payload data - static TArrayRef<const char> GetPayload(TArrayRef<const char> data) { - return data.Slice(sizeof(TBusHeader)); - } - - private: - void DoReset(); - - /// serialize message identity to be used to construct reply message - void GetIdentity(TBusIdentity& ident) const; - - /// set message identity from serialized form - void SetIdentity(const TBusIdentity& ident); - - public: - TNetAddr GetReplyTo() const { - return ReplyTo.ToNetAddr(); - } - - /// store of application specific data, never serialized into wire - void* Data; - }; - - class TBusMessageAutoPtr: public TAutoPtr<TBusMessage> { - public: - TBusMessageAutoPtr() { - } - - TBusMessageAutoPtr(TBusMessage* message) - : TAutoPtr<TBusMessage>(message) - { - } - - template <typename T1> - TBusMessageAutoPtr(const TAutoPtr<T1>& that) - : TAutoPtr<TBusMessage>(that.Release()) - { - } - }; - -} +#define TBUSMAX_MESSAGE 26 * 1024 * 1024 + sizeof(NBus::TBusHeader) ///< is't it enough? +#define TBUSMIN_MESSAGE sizeof(NBus::TBusHeader) ///< can't be less then header + + inline bool IsVersionNegotiation(const NBus::TBusHeader& header) { + return header.Id == 0 && header.Size == sizeof(TBusHeader); + } + + ////////////////////////////////////////////////////////// + /// \brief Base class for all messages passed in the system + + enum ECreateUninitialized { + MESSAGE_CREATE_UNINITIALIZED, + }; + + class TBusMessage + : protected TBusHeader, + public TRefCounted<TBusMessage, TAtomicCounter, TDelete>, + private TNonCopyable { + friend class TLocalSession; + friend struct ::NBus::NPrivate::TBusSessionImpl; + friend class ::NBus::NPrivate::TRemoteServerSession; + friend class ::NBus::NPrivate::TRemoteClientSession; + friend class ::NBus::NPrivate::TRemoteConnection; + friend class ::NBus::NPrivate::TRemoteClientConnection; + friend class ::NBus::NPrivate::TRemoteServerConnection; + friend struct ::NBus::NPrivate::TBusMessagePtrAndHeader; + + private: + ui32 LocalFlags; + + /// connection identity for reply set by PushMessage() + NPrivate::TBusSocketAddr ReplyTo; + // server-side response only, hack + ui32 RequestSize; + + TInstant RecvTime; + + public: + /// constructor to create messages on sending end + TBusMessage(ui16 type, int approxsize = sizeof(TBusHeader)); + + /// constructor with serialzed data to examine the header + TBusMessage(ECreateUninitialized); + + // slow, for diagnostics only + virtual TString Describe() const; + + // must be called if this message object needs to be reused + void Reset(); + + void CheckClean() const; + + void SetCompressed(bool); + void SetCompressedResponse(bool); + + private: + bool IsCompressed() const { + return FlagsInternal & MESSAGE_COMPRESS_INTERNAL; + } + bool IsCompressedResponse() const { + return FlagsInternal & MESSAGE_COMPRESS_RESPONSE; + } + + public: + /// can have private data to destroy + virtual ~TBusMessage(); + + /// returns header of the message + TBusHeader* GetHeader() { + return this; + } + const TBusHeader* GetHeader() const { + return this; + } + + /// helper to return type for protocol object to unpack object + static ui16 GetType(TArrayRef<const char> data) { + return TBusHeader(data).Type; + } + + /// returns payload data + static TArrayRef<const char> GetPayload(TArrayRef<const char> data) { + return data.Slice(sizeof(TBusHeader)); + } + + private: + void DoReset(); + + /// serialize message identity to be used to construct reply message + void GetIdentity(TBusIdentity& ident) const; + + /// set message identity from serialized form + void SetIdentity(const TBusIdentity& ident); + + public: + TNetAddr GetReplyTo() const { + return ReplyTo.ToNetAddr(); + } + + /// store of application specific data, never serialized into wire + void* Data; + }; + + class TBusMessageAutoPtr: public TAutoPtr<TBusMessage> { + public: + TBusMessageAutoPtr() { + } + + TBusMessageAutoPtr(TBusMessage* message) + : TAutoPtr<TBusMessage>(message) + { + } + + template <typename T1> + TBusMessageAutoPtr(const TAutoPtr<T1>& that) + : TAutoPtr<TBusMessage>(that.Release()) + { + } + }; + +} diff --git a/library/cpp/messagebus/message_counter.cpp b/library/cpp/messagebus/message_counter.cpp index 04d9343f6a..1a945eecf5 100644 --- a/library/cpp/messagebus/message_counter.cpp +++ b/library/cpp/messagebus/message_counter.cpp @@ -11,8 +11,8 @@ TMessageCounter::TMessageCounter() , Count(0) , CountCompressed(0) , CountCompressionRequests(0) -{ -} +{ +} TMessageCounter& TMessageCounter::operator+=(const TMessageCounter& that) { BytesData += that.BytesData; diff --git a/library/cpp/messagebus/message_counter.h b/library/cpp/messagebus/message_counter.h index e4be1180b0..8723e56c54 100644 --- a/library/cpp/messagebus/message_counter.h +++ b/library/cpp/messagebus/message_counter.h @@ -4,33 +4,33 @@ #include <cstddef> -namespace NBus { - namespace NPrivate { - struct TMessageCounter { - size_t BytesData; - size_t BytesNetwork; - size_t Count; - size_t CountCompressed; - size_t CountCompressionRequests; // reader only - - void AddMessage(size_t bytesData, size_t bytesCompressed, bool Compressed, bool compressionRequested) { - BytesData += bytesData; - BytesNetwork += bytesCompressed; - Count += 1; - if (Compressed) { - CountCompressed += 1; - } - if (compressionRequested) { - CountCompressionRequests += 1; - } - } - - TMessageCounter& operator+=(const TMessageCounter& that); - - TString ToString(bool reader) const; - - TMessageCounter(); - }; - +namespace NBus { + namespace NPrivate { + struct TMessageCounter { + size_t BytesData; + size_t BytesNetwork; + size_t Count; + size_t CountCompressed; + size_t CountCompressionRequests; // reader only + + void AddMessage(size_t bytesData, size_t bytesCompressed, bool Compressed, bool compressionRequested) { + BytesData += bytesData; + BytesNetwork += bytesCompressed; + Count += 1; + if (Compressed) { + CountCompressed += 1; + } + if (compressionRequested) { + CountCompressionRequests += 1; + } + } + + TMessageCounter& operator+=(const TMessageCounter& that); + + TString ToString(bool reader) const; + + TMessageCounter(); + }; + } -} +} diff --git a/library/cpp/messagebus/message_ptr_and_header.h b/library/cpp/messagebus/message_ptr_and_header.h index 9b4e2fd270..0974e8244e 100644 --- a/library/cpp/messagebus/message_ptr_and_header.h +++ b/library/cpp/messagebus/message_ptr_and_header.h @@ -6,31 +6,31 @@ #include <util/generic/noncopyable.h> #include <util/generic/utility.h> -namespace NBus { - namespace NPrivate { - struct TBusMessagePtrAndHeader : TNonCopyable { - TNonDestroyingHolder<TBusMessage> MessagePtr; - TBusHeader Header; - ui32 LocalFlags; +namespace NBus { + namespace NPrivate { + struct TBusMessagePtrAndHeader : TNonCopyable { + TNonDestroyingHolder<TBusMessage> MessagePtr; + TBusHeader Header; + ui32 LocalFlags; - TBusMessagePtrAndHeader() - : LocalFlags() - { - } + TBusMessagePtrAndHeader() + : LocalFlags() + { + } explicit TBusMessagePtrAndHeader(TBusMessage* messagePtr) : MessagePtr(messagePtr) - , Header(*MessagePtr->GetHeader()) - , LocalFlags(MessagePtr->LocalFlags) - { - } + , Header(*MessagePtr->GetHeader()) + , LocalFlags(MessagePtr->LocalFlags) + { + } - void Swap(TBusMessagePtrAndHeader& that) { - DoSwap(MessagePtr, that.MessagePtr); - DoSwap(Header, that.Header); - DoSwap(LocalFlags, that.LocalFlags); - } - }; + void Swap(TBusMessagePtrAndHeader& that) { + DoSwap(MessagePtr, that.MessagePtr); + DoSwap(Header, that.Header); + DoSwap(LocalFlags, that.LocalFlags); + } + }; - } -} + } +} diff --git a/library/cpp/messagebus/message_status.cpp b/library/cpp/messagebus/message_status.cpp index 41ad62b73f..a18d82f305 100644 --- a/library/cpp/messagebus/message_status.cpp +++ b/library/cpp/messagebus/message_status.cpp @@ -3,9 +3,9 @@ using namespace NBus; const char* NBus::MessageStatusDescription(EMessageStatus messageStatus) { -#define MESSAGE_STATUS_DESCRIPTION_GEN(name, description, ...) \ - if (messageStatus == name) \ - return description; +#define MESSAGE_STATUS_DESCRIPTION_GEN(name, description, ...) \ + if (messageStatus == name) \ + return description; MESSAGE_STATUS_MAP(MESSAGE_STATUS_DESCRIPTION_GEN) diff --git a/library/cpp/messagebus/message_status.h b/library/cpp/messagebus/message_status.h index e1878960b3..f449004e96 100644 --- a/library/cpp/messagebus/message_status.h +++ b/library/cpp/messagebus/message_status.h @@ -9,49 +9,49 @@ namespace NBus { //////////////////////////////////////////////////////////////// /// \brief Status of message communication -#define MESSAGE_STATUS_MAP(XX) \ - XX(MESSAGE_OK, "OK") \ - XX(MESSAGE_CONNECT_FAILED, "Connect failed") \ - XX(MESSAGE_TIMEOUT, "Message timed out") \ - XX(MESSAGE_SERVICE_UNKNOWN, "Locator hasn't found address for key") \ - XX(MESSAGE_BUSY, "Too many messages in flight") \ - XX(MESSAGE_UNKNOWN, "Request not found by id, usually it means that message is timed out") \ - XX(MESSAGE_DESERIALIZE_ERROR, "Deserialize by TBusProtocol failed") \ - XX(MESSAGE_HEADER_CORRUPTED, "Header corrupted") \ - XX(MESSAGE_DECOMPRESS_ERROR, "Failed to decompress") \ - XX(MESSAGE_MESSAGE_TOO_LARGE, "Message too large") \ - XX(MESSAGE_REPLY_FAILED, "Unused by messagebus, used by other code") \ - XX(MESSAGE_DELIVERY_FAILED, "Message delivery failed because connection is closed") \ - XX(MESSAGE_INVALID_VERSION, "Protocol error: invalid version") \ - XX(MESSAGE_SERVICE_TOOMANY, "Locator failed to resolve address") \ - XX(MESSAGE_SHUTDOWN, "Failure because of either session or connection shutdown") \ - XX(MESSAGE_DONT_ASK, "Internal error code used by modules") - - enum EMessageStatus { - MESSAGE_STATUS_MAP(ENUM_VALUE_GEN_NO_VALUE) - MESSAGE_STATUS_COUNT - }; - - ENUM_TO_STRING(EMessageStatus, MESSAGE_STATUS_MAP) - - const char* MessageStatusDescription(EMessageStatus); - - static inline const char* GetMessageStatus(EMessageStatus status) { - return ToCString(status); - } - - // For lwtrace - struct TMessageStatusField { - typedef int TStoreType; +#define MESSAGE_STATUS_MAP(XX) \ + XX(MESSAGE_OK, "OK") \ + XX(MESSAGE_CONNECT_FAILED, "Connect failed") \ + XX(MESSAGE_TIMEOUT, "Message timed out") \ + XX(MESSAGE_SERVICE_UNKNOWN, "Locator hasn't found address for key") \ + XX(MESSAGE_BUSY, "Too many messages in flight") \ + XX(MESSAGE_UNKNOWN, "Request not found by id, usually it means that message is timed out") \ + XX(MESSAGE_DESERIALIZE_ERROR, "Deserialize by TBusProtocol failed") \ + XX(MESSAGE_HEADER_CORRUPTED, "Header corrupted") \ + XX(MESSAGE_DECOMPRESS_ERROR, "Failed to decompress") \ + XX(MESSAGE_MESSAGE_TOO_LARGE, "Message too large") \ + XX(MESSAGE_REPLY_FAILED, "Unused by messagebus, used by other code") \ + XX(MESSAGE_DELIVERY_FAILED, "Message delivery failed because connection is closed") \ + XX(MESSAGE_INVALID_VERSION, "Protocol error: invalid version") \ + XX(MESSAGE_SERVICE_TOOMANY, "Locator failed to resolve address") \ + XX(MESSAGE_SHUTDOWN, "Failure because of either session or connection shutdown") \ + XX(MESSAGE_DONT_ASK, "Internal error code used by modules") + + enum EMessageStatus { + MESSAGE_STATUS_MAP(ENUM_VALUE_GEN_NO_VALUE) + MESSAGE_STATUS_COUNT + }; + + ENUM_TO_STRING(EMessageStatus, MESSAGE_STATUS_MAP) + + const char* MessageStatusDescription(EMessageStatus); + + static inline const char* GetMessageStatus(EMessageStatus status) { + return ToCString(status); + } + + // For lwtrace + struct TMessageStatusField { + typedef int TStoreType; typedef int TFuncParam; - static void ToString(int value, TString* out) { - *out = GetMessageStatus((NBus::EMessageStatus)value); - } + static void ToString(int value, TString* out) { + *out = GetMessageStatus((NBus::EMessageStatus)value); + } static int ToStoreType(int value) { return value; } - }; + }; } // ns diff --git a/library/cpp/messagebus/message_status_counter.cpp b/library/cpp/messagebus/message_status_counter.cpp index 891c8f5bb2..c8222a475b 100644 --- a/library/cpp/messagebus/message_status_counter.cpp +++ b/library/cpp/messagebus/message_status_counter.cpp @@ -10,7 +10,7 @@ using namespace NBus; using namespace NBus::NPrivate; -TMessageStatusCounter::TMessageStatusCounter() { +TMessageStatusCounter::TMessageStatusCounter() { Zero(Counts); } diff --git a/library/cpp/messagebus/message_status_counter.h b/library/cpp/messagebus/message_status_counter.h index e8ba2fdd31..f54e79919b 100644 --- a/library/cpp/messagebus/message_status_counter.h +++ b/library/cpp/messagebus/message_status_counter.h @@ -8,29 +8,29 @@ #include <array> -namespace NBus { - namespace NPrivate { - struct TMessageStatusCounter { - static TMessageStatusRecord::EMessageStatus MessageStatusToProtobuf(EMessageStatus status) { - return (TMessageStatusRecord::EMessageStatus)status; - } +namespace NBus { + namespace NPrivate { + struct TMessageStatusCounter { + static TMessageStatusRecord::EMessageStatus MessageStatusToProtobuf(EMessageStatus status) { + return (TMessageStatusRecord::EMessageStatus)status; + } - std::array<unsigned, MESSAGE_STATUS_COUNT> Counts; + std::array<unsigned, MESSAGE_STATUS_COUNT> Counts; - unsigned& operator[](EMessageStatus index) { - return Counts[index]; - } - const unsigned& operator[](EMessageStatus index) const { - return Counts[index]; - } + unsigned& operator[](EMessageStatus index) { + return Counts[index]; + } + const unsigned& operator[](EMessageStatus index) const { + return Counts[index]; + } - TMessageStatusCounter(); + TMessageStatusCounter(); - TMessageStatusCounter& operator+=(const TMessageStatusCounter&); + TMessageStatusCounter& operator+=(const TMessageStatusCounter&); - TString PrintToString() const; - void FillErrorsProtobuf(TConnectionStatusMonRecord*) const; - }; + TString PrintToString() const; + void FillErrorsProtobuf(TConnectionStatusMonRecord*) const; + }; - } -} + } +} diff --git a/library/cpp/messagebus/messqueue.cpp b/library/cpp/messagebus/messqueue.cpp index 3474d62705..e7ee811fd4 100644 --- a/library/cpp/messagebus/messqueue.cpp +++ b/library/cpp/messagebus/messqueue.cpp @@ -59,7 +59,7 @@ TBusMessageQueue::~TBusMessageQueue() { Stop(); } -void TBusMessageQueue::Stop() { +void TBusMessageQueue::Stop() { if (!AtomicCas(&Running, 0, 1)) { ShutdownComplete.WaitI(); return; @@ -98,14 +98,14 @@ TString TBusMessageQueue::GetStatus(ui16 flags) const { ss << GetStatusSelf(); - TList<TIntrusivePtr<TBusSessionImpl>> sessions; + TList<TIntrusivePtr<TBusSessionImpl>> sessions; { TGuard<TMutex> scope(Lock); sessions = Sessions; } - for (TList<TIntrusivePtr<TBusSessionImpl>>::const_iterator session = sessions.begin(); - session != sessions.end(); ++session) { + for (TList<TIntrusivePtr<TBusSessionImpl>>::const_iterator session = sessions.begin(); + session != sessions.end(); ++session) { ss << Endl; ss << (*session)->GetStatus(flags); } @@ -124,13 +124,13 @@ TString TBusMessageQueue::GetStatus(ui16 flags) const { return ss.Str(); } -TBusClientSessionPtr TBusMessageQueue::CreateSource(TBusProtocol* proto, IBusClientHandler* handler, const TBusClientSessionConfig& config, const TString& name) { +TBusClientSessionPtr TBusMessageQueue::CreateSource(TBusProtocol* proto, IBusClientHandler* handler, const TBusClientSessionConfig& config, const TString& name) { TRemoteClientSessionPtr session(new TRemoteClientSession(this, proto, handler, config, name)); Add(session.Get()); return session.Get(); } -TBusServerSessionPtr TBusMessageQueue::CreateDestination(TBusProtocol* proto, IBusServerHandler* handler, const TBusClientSessionConfig& config, const TString& name) { +TBusServerSessionPtr TBusMessageQueue::CreateDestination(TBusProtocol* proto, IBusServerHandler* handler, const TBusClientSessionConfig& config, const TString& name) { TRemoteServerSessionPtr session(new TRemoteServerSession(this, proto, handler, config, name)); try { int port = config.ListenPort; @@ -150,7 +150,7 @@ TBusServerSessionPtr TBusMessageQueue::CreateDestination(TBusProtocol* proto, IB } } -TBusServerSessionPtr TBusMessageQueue::CreateDestination(TBusProtocol* proto, IBusServerHandler* handler, const TBusServerSessionConfig& config, const TVector<TBindResult>& bindTo, const TString& name) { +TBusServerSessionPtr TBusMessageQueue::CreateDestination(TBusProtocol* proto, IBusServerHandler* handler, const TBusServerSessionConfig& config, const TVector<TBindResult>& bindTo, const TString& name) { TRemoteServerSessionPtr session(new TRemoteServerSession(this, proto, handler, config, name)); try { session->Listen(bindTo, this); @@ -162,23 +162,23 @@ TBusServerSessionPtr TBusMessageQueue::CreateDestination(TBusProtocol* proto, IB } void TBusMessageQueue::Add(TIntrusivePtr<TBusSessionImpl> session) { - TGuard<TMutex> scope(Lock); - Sessions.push_back(session); + TGuard<TMutex> scope(Lock); + Sessions.push_back(session); } void TBusMessageQueue::Remove(TBusSession* session) { TGuard<TMutex> scope(Lock); - TList<TIntrusivePtr<TBusSessionImpl>>::iterator it = std::find(Sessions.begin(), Sessions.end(), session); + TList<TIntrusivePtr<TBusSessionImpl>>::iterator it = std::find(Sessions.begin(), Sessions.end(), session); Y_VERIFY(it != Sessions.end(), "do not destroy session twice"); Sessions.erase(it); } -void TBusMessageQueue::Destroy(TBusSession* session) { +void TBusMessageQueue::Destroy(TBusSession* session) { session->Shutdown(); } void TBusMessageQueue::DestroyAllSessions() { - TList<TIntrusivePtr<TBusSessionImpl>> sessions; + TList<TIntrusivePtr<TBusSessionImpl>> sessions; { TGuard<TMutex> scope(Lock); sessions = Sessions; diff --git a/library/cpp/messagebus/misc/atomic_box.h b/library/cpp/messagebus/misc/atomic_box.h index 401621f933..c36b6c747f 100644 --- a/library/cpp/messagebus/misc/atomic_box.h +++ b/library/cpp/messagebus/misc/atomic_box.h @@ -12,23 +12,23 @@ private: T ValueForDebugger; }; - static_assert(sizeof(T) <= sizeof(TAtomic), "expect sizeof(T) <= sizeof(TAtomic)"); - + static_assert(sizeof(T) <= sizeof(TAtomic), "expect sizeof(T) <= sizeof(TAtomic)"); + public: - TAtomicBox(T value = T()) - : Value(value) - { - } + TAtomicBox(T value = T()) + : Value(value) + { + } void Set(T value) { - AtomicSet(Value, (TAtomic)value); + AtomicSet(Value, (TAtomic)value); } T Get() const { - return (T)AtomicGet(Value); + return (T)AtomicGet(Value); } bool CompareAndSet(T expected, T set) { - return AtomicCas(&Value, (TAtomicBase)set, (TAtomicBase)expected); + return AtomicCas(&Value, (TAtomicBase)set, (TAtomicBase)expected); } }; diff --git a/library/cpp/messagebus/misc/granup.h b/library/cpp/messagebus/misc/granup.h index 36ecfebc93..ff6cf522b4 100644 --- a/library/cpp/messagebus/misc/granup.h +++ b/library/cpp/messagebus/misc/granup.h @@ -12,29 +12,29 @@ namespace NBus { TGranUp(TDuration gran) : Gran(gran) , Next(TInstant::MicroSeconds(0)) - { - } + { + } - template <typename TFunctor> - void Update(TFunctor functor, TInstant now, bool force = false) { - if (force || now > Next) + template <typename TFunctor> + void Update(TFunctor functor, TInstant now, bool force = false) { + if (force || now > Next) Set(functor(), now); } - void Update(const TItem& item, TInstant now, bool force = false) { - if (force || now > Next) + void Update(const TItem& item, TInstant now, bool force = false) { + if (force || now > Next) Set(item, now); } - TItem Get() const noexcept { - TGuard<TLocker> guard(Lock); + TItem Get() const noexcept { + TGuard<TLocker> guard(Lock); return Item; } protected: - void Set(const TItem& item, TInstant now) { - TGuard<TLocker> guard(Lock); + void Set(const TItem& item, TInstant now) { + TGuard<TLocker> guard(Lock); Item = item; @@ -43,8 +43,8 @@ namespace NBus { private: const TDuration Gran; - TLocker Lock; - TItem Item; - TInstant Next; + TLocker Lock; + TItem Item; + TInstant Next; }; } diff --git a/library/cpp/messagebus/misc/test_sync.h b/library/cpp/messagebus/misc/test_sync.h index be3f4f20b8..9a6a718f7b 100644 --- a/library/cpp/messagebus/misc/test_sync.h +++ b/library/cpp/messagebus/misc/test_sync.h @@ -9,12 +9,12 @@ private: TMutex Mutex; TCondVar CondVar; - + public: TTestSync() : Current(0) - { - } + { + } void Inc() { TGuard<TMutex> guard(Mutex); diff --git a/library/cpp/messagebus/misc/tokenquota.h b/library/cpp/messagebus/misc/tokenquota.h index 190547fa54..d5bdd79625 100644 --- a/library/cpp/messagebus/misc/tokenquota.h +++ b/library/cpp/messagebus/misc/tokenquota.h @@ -25,26 +25,26 @@ namespace NBus { Y_UNUSED(padd_); } - bool Acquire(TAtomic level = 1, bool force = false) { - level = Max(TAtomicBase(level), TAtomicBase(1)); + bool Acquire(TAtomic level = 1, bool force = false) { + level = Max(TAtomicBase(level), TAtomicBase(1)); - if (Enabled && (Acquired < level || force)) { + if (Enabled && (Acquired < level || force)) { Acquired += AtomicSwap(&Tokens_, 0); } return !Enabled || Acquired >= level; } - void Consume(size_t items) { - if (Enabled) { + void Consume(size_t items) { + if (Enabled) { Y_ASSERT(Acquired >= TAtomicBase(items)); Acquired -= items; } } - bool Return(size_t items_) noexcept { - if (!Enabled || items_ == 0) + bool Return(size_t items_) noexcept { + if (!Enabled || items_ == 0) return false; const TAtomic items = items_; @@ -53,31 +53,31 @@ namespace NBus { return (value - items < WakeLev && value >= WakeLev); } - bool IsEnabled() const noexcept { + bool IsEnabled() const noexcept { return Enabled; } - bool IsAboveWake() const noexcept { + bool IsAboveWake() const noexcept { return !Enabled || (WakeLev <= AtomicGet(Tokens_)); } - size_t Tokens() const noexcept { + size_t Tokens() const noexcept { return Acquired + AtomicGet(Tokens_); } - size_t Check(const TAtomic level) const noexcept { + size_t Check(const TAtomic level) const noexcept { return !Enabled || level <= Acquired; } private: - bool Enabled; - TAtomicBase Acquired; + bool Enabled; + TAtomicBase Acquired; const TAtomicBase WakeLev; - TAtomic Tokens_; + TAtomic Tokens_; /* This padd requires for align Tokens_ member on its own CPU cacheline. */ - ui64 padd_; + ui64 padd_; }; } diff --git a/library/cpp/messagebus/misc/weak_ptr.h b/library/cpp/messagebus/misc/weak_ptr.h index 70fdeb0e2a..ac20cd7d43 100644 --- a/library/cpp/messagebus/misc/weak_ptr.h +++ b/library/cpp/messagebus/misc/weak_ptr.h @@ -8,18 +8,18 @@ struct TWeakPtr; template <typename TSelf> struct TWeakRefCounted { - template <typename> - friend struct TWeakPtr; - + template <typename> + friend struct TWeakPtr; + private: - struct TRef: public TAtomicRefCount<TRef> { + struct TRef: public TAtomicRefCount<TRef> { TMutex Mutex; TSelf* Outer; - TRef(TSelf* outer) - : Outer(outer) - { - } + TRef(TSelf* outer) + : Outer(outer) + { + } void Release() { TGuard<TMutex> g(Mutex); @@ -40,8 +40,8 @@ private: public: TWeakRefCounted() : RefPtr(new TRef(static_cast<TSelf*>(this))) - { - } + { + } void Ref() { Counter.Inc(); @@ -72,10 +72,10 @@ struct TWeakPtr { private: typedef TIntrusivePtr<typename T::TRef> TRefPtr; TRefPtr RefPtr; - + public: - TWeakPtr() { - } + TWeakPtr() { + } TWeakPtr(T* t) { if (!!t) { diff --git a/library/cpp/messagebus/misc/weak_ptr_ut.cpp b/library/cpp/messagebus/misc/weak_ptr_ut.cpp index 5a325278db..9677ae21b5 100644 --- a/library/cpp/messagebus/misc/weak_ptr_ut.cpp +++ b/library/cpp/messagebus/misc/weak_ptr_ut.cpp @@ -6,13 +6,13 @@ Y_UNIT_TEST_SUITE(TWeakPtrTest) { struct TWeakPtrTester: public TWeakRefCounted<TWeakPtrTester> { int* const CounterPtr; - TWeakPtrTester(int* counterPtr) - : CounterPtr(counterPtr) - { - } - ~TWeakPtrTester() { - ++*CounterPtr; - } + TWeakPtrTester(int* counterPtr) + : CounterPtr(counterPtr) + { + } + ~TWeakPtrTester() { + ++*CounterPtr; + } }; Y_UNIT_TEST(Simple) { diff --git a/library/cpp/messagebus/moved.h b/library/cpp/messagebus/moved.h index ede8dcd244..4490de10c3 100644 --- a/library/cpp/messagebus/moved.h +++ b/library/cpp/messagebus/moved.h @@ -6,10 +6,10 @@ template <typename T> class TMoved { private: mutable T Value; - + public: - TMoved() { - } + TMoved() { + } TMoved(const TMoved<T>& that) { DoSwap(Value, that.Value); } diff --git a/library/cpp/messagebus/moved_ut.cpp b/library/cpp/messagebus/moved_ut.cpp index c1a07cce7e..32e726fb19 100644 --- a/library/cpp/messagebus/moved_ut.cpp +++ b/library/cpp/messagebus/moved_ut.cpp @@ -5,13 +5,13 @@ Y_UNIT_TEST_SUITE(TMovedTest) { Y_UNIT_TEST(Simple) { TMoved<THolder<int>> h1(MakeHolder<int>(10)); - TMoved<THolder<int>> h2 = h1; + TMoved<THolder<int>> h2 = h1; UNIT_ASSERT(!*h1); UNIT_ASSERT(!!*h2); UNIT_ASSERT_VALUES_EQUAL(10, **h2); } - void Foo(TMoved<THolder<int>> h) { + void Foo(TMoved<THolder<int>> h) { UNIT_ASSERT_VALUES_EQUAL(11, **h); } diff --git a/library/cpp/messagebus/network.cpp b/library/cpp/messagebus/network.cpp index 304bedae5a..a4978f5bc4 100644 --- a/library/cpp/messagebus/network.cpp +++ b/library/cpp/messagebus/network.cpp @@ -41,11 +41,11 @@ namespace { socklen_t len; if (af == AF_INET) { len = sizeof(sockaddr_in); - ((sockaddr_in*)sa)->sin_port = HostToInet((ui16)port); - ((sockaddr_in*)sa)->sin_addr.s_addr = INADDR_ANY; + ((sockaddr_in*)sa)->sin_port = HostToInet((ui16)port); + ((sockaddr_in*)sa)->sin_addr.s_addr = INADDR_ANY; } else { len = sizeof(sockaddr_in6); - ((sockaddr_in6*)sa)->sin6_port = HostToInet((ui16)port); + ((sockaddr_in6*)sa)->sin6_port = HostToInet((ui16)port); } if (af == AF_INET6) { @@ -75,7 +75,7 @@ namespace { TMaybe<TBindResult> TryBindOnPortProto(int port, int af, bool reusePort) { try { - return {BindOnPortProto(port, af, reusePort)}; + return {BindOnPortProto(port, af, reusePort)}; } catch (const TSystemError&) { return {}; } @@ -99,7 +99,7 @@ std::pair<unsigned, TVector<TBindResult>> NBus::BindOnPort(int port, bool reuseP if (port != 0) { return AggregateBindResults(BindOnPortProto(port, AF_INET, reusePort), - BindOnPortProto(port, AF_INET6, reusePort)); + BindOnPortProto(port, AF_INET6, reusePort)); } // use nothrow versions in cycle @@ -124,7 +124,7 @@ std::pair<unsigned, TVector<TBindResult>> NBus::BindOnPort(int port, bool reuseP void NBus::NPrivate::SetSockOptTcpCork(SOCKET s, bool value) { #ifdef _linux_ - CheckedSetSockOpt(s, IPPROTO_TCP, TCP_CORK, (int)value, "TCP_CORK"); + CheckedSetSockOpt(s, IPPROTO_TCP, TCP_CORK, (int)value, "TCP_CORK"); #else Y_UNUSED(s); Y_UNUSED(value); diff --git a/library/cpp/messagebus/network.h b/library/cpp/messagebus/network.h index cc4bd76ea3..0a2be05dc2 100644 --- a/library/cpp/messagebus/network.h +++ b/library/cpp/messagebus/network.h @@ -9,20 +9,20 @@ #include <utility> namespace NBus { - namespace NPrivate { - void SetSockOptTcpCork(SOCKET s, bool value); + namespace NPrivate { + void SetSockOptTcpCork(SOCKET s, bool value); [[nodiscard]] ssize_t SocketSend(SOCKET s, TArrayRef<const char> data); [[nodiscard]] ssize_t SocketRecv(SOCKET s, TArrayRef<char> buffer); - } + } - struct TBindResult { - TSimpleSharedPtr<TSocketHolder> Socket; - TNetAddr Addr; - }; + struct TBindResult { + TSimpleSharedPtr<TSocketHolder> Socket; + TNetAddr Addr; + }; - std::pair<unsigned, TVector<TBindResult>> BindOnPort(int port, bool reusePort); + std::pair<unsigned, TVector<TBindResult>> BindOnPort(int port, bool reusePort); -} +} diff --git a/library/cpp/messagebus/network_ut.cpp b/library/cpp/messagebus/network_ut.cpp index f1798419db..47edd1b59c 100644 --- a/library/cpp/messagebus/network_ut.cpp +++ b/library/cpp/messagebus/network_ut.cpp @@ -15,14 +15,14 @@ namespace { socklen_t len = sizeof(addr); - int r = ::getsockname(socket, (sockaddr*)&addr, &len); + int r = ::getsockname(socket, (sockaddr*)&addr, &len); UNIT_ASSERT(r >= 0); if (addr.ss_family == AF_INET) { - sockaddr_in* addr_in = (sockaddr_in*)&addr; + sockaddr_in* addr_in = (sockaddr_in*)&addr; return InetToHost(addr_in->sin_port); } else if (addr.ss_family == AF_INET6) { - sockaddr_in6* addr_in6 = (sockaddr_in6*)&addr; + sockaddr_in6* addr_in6 = (sockaddr_in6*)&addr; return InetToHost(addr_in6->sin6_port); } else { UNIT_FAIL("unknown AF"); diff --git a/library/cpp/messagebus/oldmodule/module.cpp b/library/cpp/messagebus/oldmodule/module.cpp index 24bd778799..f423214aa3 100644 --- a/library/cpp/messagebus/oldmodule/module.cpp +++ b/library/cpp/messagebus/oldmodule/module.cpp @@ -16,680 +16,680 @@ using namespace NBus; using namespace NBus::NPrivate; namespace { - Y_POD_STATIC_THREAD(TBusJob*) - ThreadCurrentJob; - - struct TThreadCurrentJobGuard { - TBusJob* Prev; - - TThreadCurrentJobGuard(TBusJob* job) - : Prev(ThreadCurrentJob) - { - Y_ASSERT(!ThreadCurrentJob || ThreadCurrentJob == job); - ThreadCurrentJob = job; - } - ~TThreadCurrentJobGuard() { - ThreadCurrentJob = Prev; - } - }; - - void ClearState(NBus::TJobState* state) { - /// skip sendbacks handlers - if (state->Message != state->Reply) { - if (state->Message) { - delete state->Message; - state->Message = nullptr; - } - - if (state->Reply) { - delete state->Reply; - state->Reply = nullptr; - } - } - } - - void ClearJobStateVector(NBus::TJobStateVec* vec) { - Y_ASSERT(vec); - - for (auto& call : *vec) { - ClearState(&call); + Y_POD_STATIC_THREAD(TBusJob*) + ThreadCurrentJob; + + struct TThreadCurrentJobGuard { + TBusJob* Prev; + + TThreadCurrentJobGuard(TBusJob* job) + : Prev(ThreadCurrentJob) + { + Y_ASSERT(!ThreadCurrentJob || ThreadCurrentJob == job); + ThreadCurrentJob = job; + } + ~TThreadCurrentJobGuard() { + ThreadCurrentJob = Prev; + } + }; + + void ClearState(NBus::TJobState* state) { + /// skip sendbacks handlers + if (state->Message != state->Reply) { + if (state->Message) { + delete state->Message; + state->Message = nullptr; + } + + if (state->Reply) { + delete state->Reply; + state->Reply = nullptr; + } + } + } + + void ClearJobStateVector(NBus::TJobStateVec* vec) { + Y_ASSERT(vec); + + for (auto& call : *vec) { + ClearState(&call); } - vec->clear(); + vec->clear(); } } namespace NBus { - namespace NPrivate { - class TJobStorage { - }; - - struct TModuleClientHandler - : public IBusClientHandler { - TModuleClientHandler(TBusModuleImpl* module) - : Module(module) - { + namespace NPrivate { + class TJobStorage { + }; + + struct TModuleClientHandler + : public IBusClientHandler { + TModuleClientHandler(TBusModuleImpl* module) + : Module(module) + { + } + + void OnReply(TAutoPtr<TBusMessage> req, TAutoPtr<TBusMessage> reply) override; + void OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage) override; + void OnError(TAutoPtr<TBusMessage> msg, EMessageStatus status) override; + void OnClientConnectionEvent(const TClientConnectionEvent& event) override; + + TBusModuleImpl* const Module; + }; + + struct TModuleServerHandler + : public IBusServerHandler { + TModuleServerHandler(TBusModuleImpl* module) + : Module(module) + { + } + + void OnMessage(TOnMessageContext& msg) override; + + TBusModuleImpl* const Module; + }; + + struct TBusModuleImpl: public TBusModuleInternal { + TBusModule* const Module; + + TBusMessageQueue* Queue; + + TScheduler Scheduler; + + const char* const Name; + + typedef TList<TJobRunner*> TBusJobList; + /// jobs currently in-flight on this module + TBusJobList Jobs; + /// module level mutex + TMutex Lock; + TCondVar ShutdownCondVar; + TAtomic JobCount; + + enum EState { + CREATED, + RUNNING, + STOPPED, + }; + + TAtomic State; + TBusModuleConfig ModuleConfig; + TBusServerSessionPtr ExternalSession; + /// protocol for local proxy session + THolder<IBusClientHandler> ModuleClientHandler; + THolder<IBusServerHandler> ModuleServerHandler; + TVector<TSimpleSharedPtr<TBusStarter>> Starters; + + // Sessions must be destroyed before + // ModuleClientHandler / ModuleServerHandler + TVector<TBusClientSessionPtr> ClientSessions; + TVector<TBusServerSessionPtr> ServerSessions; + + TBusModuleImpl(TBusModule* module, const char* name) + : Module(module) + , Queue() + , Name(name) + , JobCount(0) + , State(CREATED) + , ExternalSession(nullptr) + , ModuleClientHandler(new TModuleClientHandler(this)) + , ModuleServerHandler(new TModuleServerHandler(this)) + { + } + + ~TBusModuleImpl() override { + // Shutdown cannot be called from destructor, + // because module has virtual methods. + Y_VERIFY(State != RUNNING, "if running, must explicitly call Shutdown() before destructor"); + + Scheduler.Stop(); + + while (!Jobs.empty()) { + DestroyJob(Jobs.front()); + } + Y_VERIFY(JobCount == 0, "state check"); + } + + void OnMessageReceived(TAutoPtr<TBusMessage> msg, TOnMessageContext&); + + void AddJob(TJobRunner* jobRunner); + + void DestroyJob(TJobRunner* job); + + /// terminate job on this message + void CancelJob(TBusJob* job, EMessageStatus status); + /// prints statuses of jobs + TString GetStatus(unsigned flags); + + size_t Size() const { + return AtomicGet(JobCount); + } + + void Shutdown(); + + TVector<TBusClientSessionPtr> GetClientSessionsInternal() override { + return ClientSessions; + } + + TVector<TBusServerSessionPtr> GetServerSessionsInternal() override { + return ServerSessions; + } + + TBusMessageQueue* GetQueue() override { + return Queue; + } + + TString GetNameInternal() override { + return Name; + } + + TString GetStatusSingleLine() override { + TStringStream ss; + ss << "jobs: " << Size(); + return ss.Str(); + } + + void OnClientConnectionEvent(const TClientConnectionEvent& event) { + Module->OnClientConnectionEvent(event); + } + }; + + struct TJobResponseMessage { + TBusMessage* Request; + TBusMessage* Response; + EMessageStatus Status; + + TJobResponseMessage(TBusMessage* request, TBusMessage* response, EMessageStatus status) + : Request(request) + , Response(response) + , Status(status) + { + } + }; + + struct TJobRunner: public TAtomicRefCount<TJobRunner>, + public NActor::TActor<TJobRunner>, + public NActor::TQueueInActor<TJobRunner, TJobResponseMessage>, + public TScheduleActor<TJobRunner> { + THolder<TBusJob> Job; + + TList<TJobRunner*>::iterator JobStorageIterator; + + TJobRunner(TAutoPtr<TBusJob> job) + : NActor::TActor<TJobRunner>(job->ModuleImpl->Queue->GetExecutor()) + , TScheduleActor<TJobRunner>(&job->ModuleImpl->Scheduler) + , Job(job.Release()) + , JobStorageIterator() + { + Job->Runner = this; + } + + ~TJobRunner() override { + Y_ASSERT(JobStorageIterator == TList<TJobRunner*>::iterator()); + } + + void ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, const TJobResponseMessage& message) { + Job->CallReplyHandler(message.Status, message.Request, message.Response); + } + + void Destroy() { + if (!!Job->OnMessageContext) { + if (!Job->ReplySent) { + Job->OnMessageContext.ForgetRequest(); + } + } + Job->ModuleImpl->DestroyJob(this); } - void OnReply(TAutoPtr<TBusMessage> req, TAutoPtr<TBusMessage> reply) override; - void OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage) override; - void OnError(TAutoPtr<TBusMessage> msg, EMessageStatus status) override; - void OnClientConnectionEvent(const TClientConnectionEvent& event) override; - - TBusModuleImpl* const Module; - }; - - struct TModuleServerHandler - : public IBusServerHandler { - TModuleServerHandler(TBusModuleImpl* module) - : Module(module) - { - } - - void OnMessage(TOnMessageContext& msg) override; - - TBusModuleImpl* const Module; - }; - - struct TBusModuleImpl: public TBusModuleInternal { - TBusModule* const Module; - - TBusMessageQueue* Queue; - - TScheduler Scheduler; - - const char* const Name; - - typedef TList<TJobRunner*> TBusJobList; - /// jobs currently in-flight on this module - TBusJobList Jobs; - /// module level mutex - TMutex Lock; - TCondVar ShutdownCondVar; - TAtomic JobCount; - - enum EState { - CREATED, - RUNNING, - STOPPED, - }; - - TAtomic State; - TBusModuleConfig ModuleConfig; - TBusServerSessionPtr ExternalSession; - /// protocol for local proxy session - THolder<IBusClientHandler> ModuleClientHandler; - THolder<IBusServerHandler> ModuleServerHandler; - TVector<TSimpleSharedPtr<TBusStarter>> Starters; - - // Sessions must be destroyed before - // ModuleClientHandler / ModuleServerHandler - TVector<TBusClientSessionPtr> ClientSessions; - TVector<TBusServerSessionPtr> ServerSessions; - - TBusModuleImpl(TBusModule* module, const char* name) - : Module(module) - , Queue() - , Name(name) - , JobCount(0) - , State(CREATED) - , ExternalSession(nullptr) - , ModuleClientHandler(new TModuleClientHandler(this)) - , ModuleServerHandler(new TModuleServerHandler(this)) - { - } - - ~TBusModuleImpl() override { - // Shutdown cannot be called from destructor, - // because module has virtual methods. - Y_VERIFY(State != RUNNING, "if running, must explicitly call Shutdown() before destructor"); - - Scheduler.Stop(); - - while (!Jobs.empty()) { - DestroyJob(Jobs.front()); + void Act(NActor::TDefaultTag) { + if (JobStorageIterator == TList<TJobRunner*>::iterator()) { + return; + } + + if (Job->SleepUntil != 0) { + if (AtomicGet(Job->ModuleImpl->State) == TBusModuleImpl::STOPPED) { + Destroy(); + return; + } + } + + TThreadCurrentJobGuard g(Job.Get()); + + NActor::TQueueInActor<TJobRunner, TJobResponseMessage>::DequeueAll(); + + if (Alarm.FetchTask()) { + if (Job->AnyPendingToSend()) { + Y_ASSERT(Job->SleepUntil == 0); + Job->SendPending(); + if (Job->AnyPendingToSend()) { + } + } else { + // regular alarm + Y_ASSERT(Job->Pending.empty()); + Y_ASSERT(Job->SleepUntil != 0); + Job->SleepUntil = 0; + } } - Y_VERIFY(JobCount == 0, "state check"); - } - void OnMessageReceived(TAutoPtr<TBusMessage> msg, TOnMessageContext&); + for (;;) { + if (Job->Pending.empty() && !!Job->Handler && Job->Status == MESSAGE_OK) { + TWhatThreadDoesPushPop pp("do call job handler (do not confuse with reply handler)"); - void AddJob(TJobRunner* jobRunner); + Job->Handler = Job->Handler(Job->Module, Job.Get(), Job->Message); + } - void DestroyJob(TJobRunner* job); + if (Job->SleepUntil != 0) { + ScheduleAt(TInstant::MilliSeconds(Job->SleepUntil)); + return; + } - /// terminate job on this message - void CancelJob(TBusJob* job, EMessageStatus status); - /// prints statuses of jobs - TString GetStatus(unsigned flags); + Job->SendPending(); - size_t Size() const { - return AtomicGet(JobCount); - } + if (Job->AnyPendingToSend()) { + ScheduleAt(TInstant::Now() + TDuration::Seconds(1)); + return; + } - void Shutdown(); + if (!Job->Pending.empty()) { + // waiting replies + return; + } - TVector<TBusClientSessionPtr> GetClientSessionsInternal() override { - return ClientSessions; + if (Job->IsDone()) { + Destroy(); + return; + } + } } + }; + + } + + static inline TJobRunner* GetJob(TBusMessage* message) { + return (TJobRunner*)message->Data; + } + + static inline void SetJob(TBusMessage* message, TJobRunner* job) { + message->Data = job; + } + + TBusJob::TBusJob(TBusModule* module, TBusMessage* message) + : Status(MESSAGE_OK) + , Runner() + , Message(message) + , ReplySent(false) + , Module(module) + , ModuleImpl(module->Impl.Get()) + , SleepUntil(0) + { + Handler = TJobHandler(&TBusModule::Start); + } + + TBusJob::~TBusJob() { + Y_ASSERT(Pending.size() == 0); + //Y_ASSERT(SleepUntil == 0); + + ClearAllMessageStates(); + } + + TNetAddr TBusJob::GetPeerAddrNetAddr() const { + Y_VERIFY(!!OnMessageContext); + return OnMessageContext.GetPeerAddrNetAddr(); + } + + void TBusJob::CheckThreadCurrentJob() { + Y_ASSERT(ThreadCurrentJob == this); + } + + ///////////////////////////////////////////////////////// + /// \brief Send messages in pending list + + /// If at least one message is gone return true + /// If message has not been send, move it to Finished with appropriate error code + bool TBusJob::SendPending() { + // Iterator type must be size_t, not vector::iterator, + // because `DoCallReplyHandler` may call `Send` that modifies `Pending` vector, + // that in turn invalidates iterator. + // Implementation assumes that `DoCallReplyHandler` only pushes back to `Pending` + // (not erases, and not inserts) so iteration by index is valid. + size_t it = 0; + while (it != Pending.size()) { + TJobState& call = Pending[it]; + + if (call.Status == MESSAGE_DONT_ASK) { + EMessageStatus getAddressStatus = MESSAGE_OK; + TNetAddr addr; + if (call.UseAddr) { + addr = call.Addr; + } else { + getAddressStatus = const_cast<TBusProtocol*>(call.Session->GetProto())->GetDestination(call.Session, call.Message, call.Session->GetQueue()->GetLocator(), &addr); + } + + if (getAddressStatus == MESSAGE_OK) { + // hold extra reference for each request in flight + Runner->Ref(); + + if (call.OneWay) { + call.Status = call.Session->SendMessageOneWay(call.Message, &addr); + } else { + call.Status = call.Session->SendMessage(call.Message, &addr); + } + + if (call.Status != MESSAGE_OK) { + Runner->UnRef(); + } - TVector<TBusServerSessionPtr> GetServerSessionsInternal() override { - return ServerSessions; - } - - TBusMessageQueue* GetQueue() override { - return Queue; - } - - TString GetNameInternal() override { - return Name; - } - - TString GetStatusSingleLine() override { - TStringStream ss; - ss << "jobs: " << Size(); - return ss.Str(); - } - - void OnClientConnectionEvent(const TClientConnectionEvent& event) { - Module->OnClientConnectionEvent(event); - } - }; - - struct TJobResponseMessage { - TBusMessage* Request; - TBusMessage* Response; - EMessageStatus Status; - - TJobResponseMessage(TBusMessage* request, TBusMessage* response, EMessageStatus status) - : Request(request) - , Response(response) - , Status(status) - { - } - }; - - struct TJobRunner: public TAtomicRefCount<TJobRunner>, - public NActor::TActor<TJobRunner>, - public NActor::TQueueInActor<TJobRunner, TJobResponseMessage>, - public TScheduleActor<TJobRunner> { - THolder<TBusJob> Job; - - TList<TJobRunner*>::iterator JobStorageIterator; - - TJobRunner(TAutoPtr<TBusJob> job) - : NActor::TActor<TJobRunner>(job->ModuleImpl->Queue->GetExecutor()) - , TScheduleActor<TJobRunner>(&job->ModuleImpl->Scheduler) - , Job(job.Release()) - , JobStorageIterator() - { - Job->Runner = this; - } - - ~TJobRunner() override { - Y_ASSERT(JobStorageIterator == TList<TJobRunner*>::iterator()); - } - - void ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, const TJobResponseMessage& message) { - Job->CallReplyHandler(message.Status, message.Request, message.Response); - } - - void Destroy() { - if (!!Job->OnMessageContext) { - if (!Job->ReplySent) { - Job->OnMessageContext.ForgetRequest(); - } - } - Job->ModuleImpl->DestroyJob(this); - } - - void Act(NActor::TDefaultTag) { - if (JobStorageIterator == TList<TJobRunner*>::iterator()) { - return; - } - - if (Job->SleepUntil != 0) { - if (AtomicGet(Job->ModuleImpl->State) == TBusModuleImpl::STOPPED) { - Destroy(); - return; - } - } - - TThreadCurrentJobGuard g(Job.Get()); - - NActor::TQueueInActor<TJobRunner, TJobResponseMessage>::DequeueAll(); - - if (Alarm.FetchTask()) { - if (Job->AnyPendingToSend()) { - Y_ASSERT(Job->SleepUntil == 0); - Job->SendPending(); - if (Job->AnyPendingToSend()) { - } - } else { - // regular alarm - Y_ASSERT(Job->Pending.empty()); - Y_ASSERT(Job->SleepUntil != 0); - Job->SleepUntil = 0; - } - } - - for (;;) { - if (Job->Pending.empty() && !!Job->Handler && Job->Status == MESSAGE_OK) { - TWhatThreadDoesPushPop pp("do call job handler (do not confuse with reply handler)"); - - Job->Handler = Job->Handler(Job->Module, Job.Get(), Job->Message); - } - - if (Job->SleepUntil != 0) { - ScheduleAt(TInstant::MilliSeconds(Job->SleepUntil)); - return; - } - - Job->SendPending(); - - if (Job->AnyPendingToSend()) { - ScheduleAt(TInstant::Now() + TDuration::Seconds(1)); - return; - } - - if (!Job->Pending.empty()) { - // waiting replies - return; - } - - if (Job->IsDone()) { - Destroy(); - return; - } - } - } - }; - - } - - static inline TJobRunner* GetJob(TBusMessage* message) { - return (TJobRunner*)message->Data; - } - - static inline void SetJob(TBusMessage* message, TJobRunner* job) { - message->Data = job; - } - - TBusJob::TBusJob(TBusModule* module, TBusMessage* message) - : Status(MESSAGE_OK) - , Runner() - , Message(message) - , ReplySent(false) - , Module(module) - , ModuleImpl(module->Impl.Get()) - , SleepUntil(0) - { - Handler = TJobHandler(&TBusModule::Start); - } - - TBusJob::~TBusJob() { - Y_ASSERT(Pending.size() == 0); - //Y_ASSERT(SleepUntil == 0); - - ClearAllMessageStates(); - } - - TNetAddr TBusJob::GetPeerAddrNetAddr() const { - Y_VERIFY(!!OnMessageContext); - return OnMessageContext.GetPeerAddrNetAddr(); - } - - void TBusJob::CheckThreadCurrentJob() { - Y_ASSERT(ThreadCurrentJob == this); - } - - ///////////////////////////////////////////////////////// - /// \brief Send messages in pending list - - /// If at least one message is gone return true - /// If message has not been send, move it to Finished with appropriate error code - bool TBusJob::SendPending() { - // Iterator type must be size_t, not vector::iterator, - // because `DoCallReplyHandler` may call `Send` that modifies `Pending` vector, - // that in turn invalidates iterator. - // Implementation assumes that `DoCallReplyHandler` only pushes back to `Pending` - // (not erases, and not inserts) so iteration by index is valid. - size_t it = 0; - while (it != Pending.size()) { - TJobState& call = Pending[it]; - - if (call.Status == MESSAGE_DONT_ASK) { - EMessageStatus getAddressStatus = MESSAGE_OK; - TNetAddr addr; - if (call.UseAddr) { - addr = call.Addr; } else { - getAddressStatus = const_cast<TBusProtocol*>(call.Session->GetProto())->GetDestination(call.Session, call.Message, call.Session->GetQueue()->GetLocator(), &addr); + call.Status = getAddressStatus; } - - if (getAddressStatus == MESSAGE_OK) { - // hold extra reference for each request in flight - Runner->Ref(); - - if (call.OneWay) { - call.Status = call.Session->SendMessageOneWay(call.Message, &addr); - } else { - call.Status = call.Session->SendMessage(call.Message, &addr); - } - - if (call.Status != MESSAGE_OK) { - Runner->UnRef(); - } - - } else { - call.Status = getAddressStatus; - } - } - - if (call.Status == MESSAGE_OK) { - ++it; // keep pending list until we get reply - } else if (call.Status == MESSAGE_BUSY) { - Y_FAIL("MESSAGE_BUSY is prohibited in modules. Please increase MaxInFlight"); - } else if (call.Status == MESSAGE_CONNECT_FAILED && call.NumRetries < call.MaxRetries) { - ++it; // try up to call.MaxRetries times to send message - call.NumRetries++; - DoCallReplyHandler(call); - call.Status = MESSAGE_DONT_ASK; - call.Message->Reset(); // generate new Id + } + + if (call.Status == MESSAGE_OK) { + ++it; // keep pending list until we get reply + } else if (call.Status == MESSAGE_BUSY) { + Y_FAIL("MESSAGE_BUSY is prohibited in modules. Please increase MaxInFlight"); + } else if (call.Status == MESSAGE_CONNECT_FAILED && call.NumRetries < call.MaxRetries) { + ++it; // try up to call.MaxRetries times to send message + call.NumRetries++; + DoCallReplyHandler(call); + call.Status = MESSAGE_DONT_ASK; + call.Message->Reset(); // generate new Id } else { - Finished.push_back(call); - DoCallReplyHandler(call); - Pending.erase(Pending.begin() + it); + Finished.push_back(call); + DoCallReplyHandler(call); + Pending.erase(Pending.begin() + it); } } - return Pending.size() > 0; - } - - bool TBusJob::AnyPendingToSend() { - for (unsigned i = 0; i < Pending.size(); ++i) { - if (Pending[i].Status == MESSAGE_DONT_ASK) { - return true; - } + return Pending.size() > 0; + } + + bool TBusJob::AnyPendingToSend() { + for (unsigned i = 0; i < Pending.size(); ++i) { + if (Pending[i].Status == MESSAGE_DONT_ASK) { + return true; + } } - - return false; + + return false; } - bool TBusJob::IsDone() { - bool r = (SleepUntil == 0 && Pending.size() == 0 && (Handler == nullptr || Status != MESSAGE_OK)); - return r; - } - - void TBusJob::CallJobHandlerOnly() { - TThreadCurrentJobGuard threadCurrentJobGuard(this); - TWhatThreadDoesPushPop pp("do call job handler (do not confuse with reply handler)"); - - Handler = Handler(ModuleImpl->Module, this, Message); + bool TBusJob::IsDone() { + bool r = (SleepUntil == 0 && Pending.size() == 0 && (Handler == nullptr || Status != MESSAGE_OK)); + return r; } - bool TBusJob::CallJobHandler() { - /// go on as far as we can go without waiting - while (!IsDone()) { - /// call the handler - CallJobHandlerOnly(); + void TBusJob::CallJobHandlerOnly() { + TThreadCurrentJobGuard threadCurrentJobGuard(this); + TWhatThreadDoesPushPop pp("do call job handler (do not confuse with reply handler)"); - /// quit if job is canceled - if (Status != MESSAGE_OK) { - break; - } + Handler = Handler(ModuleImpl->Module, this, Message); + } - /// there are messages to send and wait for reply - SendPending(); + bool TBusJob::CallJobHandler() { + /// go on as far as we can go without waiting + while (!IsDone()) { + /// call the handler + CallJobHandlerOnly(); + + /// quit if job is canceled + if (Status != MESSAGE_OK) { + break; + } - if (!Pending.empty()) { - break; - } + /// there are messages to send and wait for reply + SendPending(); - /// asked to sleep - if (SleepUntil) { - break; - } - } + if (!Pending.empty()) { + break; + } - Y_VERIFY(!(Pending.size() == 0 && Handler == nullptr && Status == MESSAGE_OK && !ReplySent), - "Handler returned NULL without Cancel() or SendReply() for message=%016" PRIx64 " type=%d", - Message->GetHeader()->Id, Message->GetHeader()->Type); + /// asked to sleep + if (SleepUntil) { + break; + } + } - return IsDone(); - } + Y_VERIFY(!(Pending.size() == 0 && Handler == nullptr && Status == MESSAGE_OK && !ReplySent), + "Handler returned NULL without Cancel() or SendReply() for message=%016" PRIx64 " type=%d", + Message->GetHeader()->Id, Message->GetHeader()->Type); - void TBusJob::DoCallReplyHandler(TJobState& call) { - if (call.Handler) { - TWhatThreadDoesPushPop pp("do call reply handler (do not confuse with job handler)"); + return IsDone(); + } - TThreadCurrentJobGuard threadCurrentJobGuard(this); - (Module->*(call.Handler))(this, call.Status, call.Message, call.Reply); - } - } + void TBusJob::DoCallReplyHandler(TJobState& call) { + if (call.Handler) { + TWhatThreadDoesPushPop pp("do call reply handler (do not confuse with job handler)"); - int TBusJob::CallReplyHandler(EMessageStatus status, TBusMessage* mess, TBusMessage* reply) { - /// find handler for given message and update it's status - size_t i = 0; - for (; i < Pending.size(); ++i) { - TJobState& call = Pending[i]; - if (call.Message == mess) { - break; - } + TThreadCurrentJobGuard threadCurrentJobGuard(this); + (Module->*(call.Handler))(this, call.Status, call.Message, call.Reply); } - - /// if not found, report error - if (i == Pending.size()) { - Y_FAIL("must not happen"); + } + + int TBusJob::CallReplyHandler(EMessageStatus status, TBusMessage* mess, TBusMessage* reply) { + /// find handler for given message and update it's status + size_t i = 0; + for (; i < Pending.size(); ++i) { + TJobState& call = Pending[i]; + if (call.Message == mess) { + break; + } } - /// fill in response into job state - TJobState& call = Pending[i]; - call.Status = status; - Y_ASSERT(call.Message == mess); - call.Reply = reply; - - if ((status == MESSAGE_TIMEOUT || status == MESSAGE_DELIVERY_FAILED) && call.NumRetries < call.MaxRetries) { - call.NumRetries++; - call.Status = MESSAGE_DONT_ASK; - call.Message->Reset(); // generate new Id - DoCallReplyHandler(call); - return 0; + /// if not found, report error + if (i == Pending.size()) { + Y_FAIL("must not happen"); + } + + /// fill in response into job state + TJobState& call = Pending[i]; + call.Status = status; + Y_ASSERT(call.Message == mess); + call.Reply = reply; + + if ((status == MESSAGE_TIMEOUT || status == MESSAGE_DELIVERY_FAILED) && call.NumRetries < call.MaxRetries) { + call.NumRetries++; + call.Status = MESSAGE_DONT_ASK; + call.Message->Reset(); // generate new Id + DoCallReplyHandler(call); + return 0; } - /// call the handler if provided - DoCallReplyHandler(call); + /// call the handler if provided + DoCallReplyHandler(call); - /// move job state into the finished stack - Finished.push_back(Pending[i]); - Pending.erase(Pending.begin() + i); + /// move job state into the finished stack + Finished.push_back(Pending[i]); + Pending.erase(Pending.begin() + i); return 0; } - /////////////////////////////////////////////////////////////// - /// send message to any other session or application - void TBusJob::Send(TBusMessageAutoPtr mess, TBusClientSession* session, TReplyHandler rhandler, size_t maxRetries) { - CheckThreadCurrentJob(); + /////////////////////////////////////////////////////////////// + /// send message to any other session or application + void TBusJob::Send(TBusMessageAutoPtr mess, TBusClientSession* session, TReplyHandler rhandler, size_t maxRetries) { + CheckThreadCurrentJob(); - SetJob(mess.Get(), Runner); - Pending.push_back(TJobState(rhandler, MESSAGE_DONT_ASK, mess.Release(), session, nullptr, maxRetries, nullptr, false)); - } + SetJob(mess.Get(), Runner); + Pending.push_back(TJobState(rhandler, MESSAGE_DONT_ASK, mess.Release(), session, nullptr, maxRetries, nullptr, false)); + } - void TBusJob::Send(TBusMessageAutoPtr mess, TBusClientSession* session, TReplyHandler rhandler, size_t maxRetries, const TNetAddr& addr) { - CheckThreadCurrentJob(); + void TBusJob::Send(TBusMessageAutoPtr mess, TBusClientSession* session, TReplyHandler rhandler, size_t maxRetries, const TNetAddr& addr) { + CheckThreadCurrentJob(); - SetJob(mess.Get(), Runner); - Pending.push_back(TJobState(rhandler, MESSAGE_DONT_ASK, mess.Release(), session, nullptr, maxRetries, &addr, false)); - } + SetJob(mess.Get(), Runner); + Pending.push_back(TJobState(rhandler, MESSAGE_DONT_ASK, mess.Release(), session, nullptr, maxRetries, &addr, false)); + } - void TBusJob::SendOneWayTo(TBusMessageAutoPtr req, TBusClientSession* session, const TNetAddr& addr) { - CheckThreadCurrentJob(); + void TBusJob::SendOneWayTo(TBusMessageAutoPtr req, TBusClientSession* session, const TNetAddr& addr) { + CheckThreadCurrentJob(); - SetJob(req.Get(), Runner); - Pending.push_back(TJobState(nullptr, MESSAGE_DONT_ASK, req.Release(), session, nullptr, 0, &addr, true)); - } + SetJob(req.Get(), Runner); + Pending.push_back(TJobState(nullptr, MESSAGE_DONT_ASK, req.Release(), session, nullptr, 0, &addr, true)); + } - void TBusJob::SendOneWayWithLocator(TBusMessageAutoPtr req, TBusClientSession* session) { - CheckThreadCurrentJob(); + void TBusJob::SendOneWayWithLocator(TBusMessageAutoPtr req, TBusClientSession* session) { + CheckThreadCurrentJob(); - SetJob(req.Get(), Runner); - Pending.push_back(TJobState(nullptr, MESSAGE_DONT_ASK, req.Release(), session, nullptr, 0, nullptr, true)); - } + SetJob(req.Get(), Runner); + Pending.push_back(TJobState(nullptr, MESSAGE_DONT_ASK, req.Release(), session, nullptr, 0, nullptr, true)); + } - /////////////////////////////////////////////////////////////// - /// send reply to the starter message - void TBusJob::SendReply(TBusMessageAutoPtr reply) { - CheckThreadCurrentJob(); + /////////////////////////////////////////////////////////////// + /// send reply to the starter message + void TBusJob::SendReply(TBusMessageAutoPtr reply) { + CheckThreadCurrentJob(); - Y_VERIFY(!ReplySent, "cannot call SendReply twice"); - ReplySent = true; - if (!OnMessageContext) - return; + Y_VERIFY(!ReplySent, "cannot call SendReply twice"); + ReplySent = true; + if (!OnMessageContext) + return; - EMessageStatus ok = OnMessageContext.SendReplyMove(reply); - if (ok != MESSAGE_OK) { - // TODO: count errors - } - } + EMessageStatus ok = OnMessageContext.SendReplyMove(reply); + if (ok != MESSAGE_OK) { + // TODO: count errors + } + } - /// set the flag to terminate job at the earliest convenience - void TBusJob::Cancel(EMessageStatus status) { - CheckThreadCurrentJob(); + /// set the flag to terminate job at the earliest convenience + void TBusJob::Cancel(EMessageStatus status) { + CheckThreadCurrentJob(); - Status = status; + Status = status; } - void TBusJob::ClearState(TJobState& call) { - TJobStateVec::iterator it; - for (it = Finished.begin(); it != Finished.end(); ++it) { - TJobState& state = *it; - if (&call == &state) { - ::ClearState(&call); - Finished.erase(it); - return; - } + void TBusJob::ClearState(TJobState& call) { + TJobStateVec::iterator it; + for (it = Finished.begin(); it != Finished.end(); ++it) { + TJobState& state = *it; + if (&call == &state) { + ::ClearState(&call); + Finished.erase(it); + return; + } } - Y_ASSERT(0); + Y_ASSERT(0); } - void TBusJob::ClearAllMessageStates() { - ClearJobStateVector(&Finished); - ClearJobStateVector(&Pending); - } + void TBusJob::ClearAllMessageStates() { + ClearJobStateVector(&Finished); + ClearJobStateVector(&Pending); + } - void TBusJob::Sleep(int milliSeconds) { - CheckThreadCurrentJob(); + void TBusJob::Sleep(int milliSeconds) { + CheckThreadCurrentJob(); - Y_VERIFY(Pending.empty(), "sleep is not allowed when there are pending job"); - Y_VERIFY(SleepUntil == 0, "must not override sleep"); + Y_VERIFY(Pending.empty(), "sleep is not allowed when there are pending job"); + Y_VERIFY(SleepUntil == 0, "must not override sleep"); - SleepUntil = Now() + milliSeconds; - } + SleepUntil = Now() + milliSeconds; + } - TString TBusJob::GetStatus(unsigned flags) { - TString strReturn; - strReturn += Sprintf(" job=%016" PRIx64 " type=%d sent=%d pending=%d (%d) %s\n", - Message->GetHeader()->Id, - (int)Message->GetHeader()->Type, - (int)(Now() - Message->GetHeader()->SendTime) / 1000, - (int)Pending.size(), - (int)Finished.size(), + TString TBusJob::GetStatus(unsigned flags) { + TString strReturn; + strReturn += Sprintf(" job=%016" PRIx64 " type=%d sent=%d pending=%d (%d) %s\n", + Message->GetHeader()->Id, + (int)Message->GetHeader()->Type, + (int)(Now() - Message->GetHeader()->SendTime) / 1000, + (int)Pending.size(), + (int)Finished.size(), Status != MESSAGE_OK ? ToString(Status).data() : ""); - - TJobStateVec::iterator it; - for (it = Pending.begin(); it != Pending.end(); ++it) { - TJobState& call = *it; - strReturn += call.GetStatus(flags); - } - return strReturn; - } - - TString TJobState::GetStatus(unsigned flags) { - Y_UNUSED(flags); - TString strReturn; - strReturn += Sprintf(" pending=%016" PRIx64 " type=%d (%s) sent=%d %s\n", - Message->GetHeader()->Id, - (int)Message->GetHeader()->Type, - Session->GetProto()->GetService(), - (int)(Now() - Message->GetHeader()->SendTime) / 1000, + + TJobStateVec::iterator it; + for (it = Pending.begin(); it != Pending.end(); ++it) { + TJobState& call = *it; + strReturn += call.GetStatus(flags); + } + return strReturn; + } + + TString TJobState::GetStatus(unsigned flags) { + Y_UNUSED(flags); + TString strReturn; + strReturn += Sprintf(" pending=%016" PRIx64 " type=%d (%s) sent=%d %s\n", + Message->GetHeader()->Id, + (int)Message->GetHeader()->Type, + Session->GetProto()->GetService(), + (int)(Now() - Message->GetHeader()->SendTime) / 1000, ToString(Status).data()); - return strReturn; - } - - ////////////////////////////////////////////////////////////////////// - - void TBusModuleImpl::CancelJob(TBusJob* job, EMessageStatus status) { - TWhatThreadDoesAcquireGuard<TMutex> G(Lock, "modules: acquiring lock for CancelJob"); - if (job) { - job->Cancel(status); - } - } - - TString TBusModuleImpl::GetStatus(unsigned flags) { - Y_UNUSED(flags); - TWhatThreadDoesAcquireGuard<TMutex> G(Lock, "modules: acquiring lock for GetStatus"); - TString strReturn = Sprintf("JobsInFlight=%d\n", (int)Jobs.size()); - for (auto job : Jobs) { - //strReturn += job->Job->GetStatus(flags); - Y_UNUSED(job); - strReturn += "TODO\n"; - } - return strReturn; - } - - TBusModuleConfig::TBusModuleConfig() - : StarterMaxInFlight(1000) - { - } - - TBusModuleConfig::TSecret::TSecret() - : SchedulePeriod(TDuration::Seconds(1)) - { - } - - TBusModule::TBusModule(const char* name) - : Impl(new TBusModuleImpl(this, name)) - { - } - - TBusModule::~TBusModule() { - } - - const char* TBusModule::GetName() const { - return Impl->Name; - } - - void TBusModule::SetConfig(const TBusModuleConfig& config) { - Impl->ModuleConfig = config; - } - - bool TBusModule::StartInput() { - Y_VERIFY(Impl->State == TBusModuleImpl::CREATED, "state check"); - Y_VERIFY(!!Impl->Queue, "state check"); - Impl->State = TBusModuleImpl::RUNNING; - - Y_ASSERT(!Impl->ExternalSession); - TBusServerSessionPtr extSession = CreateExtSession(*Impl->Queue); - if (extSession != nullptr) { - Impl->ExternalSession = extSession; - } - - return true; - } - - bool TBusModule::Shutdown() { - Impl->Shutdown(); - - return true; - } - - TBusJob* TBusModule::CreateJobInstance(TBusMessage* message) { - TBusJob* job = new TBusJob(this, message); - return job; - } - - /** + return strReturn; + } + + ////////////////////////////////////////////////////////////////////// + + void TBusModuleImpl::CancelJob(TBusJob* job, EMessageStatus status) { + TWhatThreadDoesAcquireGuard<TMutex> G(Lock, "modules: acquiring lock for CancelJob"); + if (job) { + job->Cancel(status); + } + } + + TString TBusModuleImpl::GetStatus(unsigned flags) { + Y_UNUSED(flags); + TWhatThreadDoesAcquireGuard<TMutex> G(Lock, "modules: acquiring lock for GetStatus"); + TString strReturn = Sprintf("JobsInFlight=%d\n", (int)Jobs.size()); + for (auto job : Jobs) { + //strReturn += job->Job->GetStatus(flags); + Y_UNUSED(job); + strReturn += "TODO\n"; + } + return strReturn; + } + + TBusModuleConfig::TBusModuleConfig() + : StarterMaxInFlight(1000) + { + } + + TBusModuleConfig::TSecret::TSecret() + : SchedulePeriod(TDuration::Seconds(1)) + { + } + + TBusModule::TBusModule(const char* name) + : Impl(new TBusModuleImpl(this, name)) + { + } + + TBusModule::~TBusModule() { + } + + const char* TBusModule::GetName() const { + return Impl->Name; + } + + void TBusModule::SetConfig(const TBusModuleConfig& config) { + Impl->ModuleConfig = config; + } + + bool TBusModule::StartInput() { + Y_VERIFY(Impl->State == TBusModuleImpl::CREATED, "state check"); + Y_VERIFY(!!Impl->Queue, "state check"); + Impl->State = TBusModuleImpl::RUNNING; + + Y_ASSERT(!Impl->ExternalSession); + TBusServerSessionPtr extSession = CreateExtSession(*Impl->Queue); + if (extSession != nullptr) { + Impl->ExternalSession = extSession; + } + + return true; + } + + bool TBusModule::Shutdown() { + Impl->Shutdown(); + + return true; + } + + TBusJob* TBusModule::CreateJobInstance(TBusMessage* message) { + TBusJob* job = new TBusJob(this, message); + return job; + } + + /** Example for external session creation: TBusSession* TMyModule::CreateExtSession(TBusMessageQueue& queue) { @@ -698,77 +698,77 @@ TBusSession* TMyModule::CreateExtSession(TBusMessageQueue& queue) { return session; */ - bool TBusModule::CreatePrivateSessions(TBusMessageQueue* queue) { - Impl->Queue = queue; - return true; - } - - int TBusModule::GetModuleSessionInFlight() const { - return Impl->Size(); - } - - TIntrusivePtr<TBusModuleInternal> TBusModule::GetInternal() { - return Impl.Get(); - } - - TBusServerSessionPtr TBusModule::CreateDefaultDestination( - TBusMessageQueue& queue, TBusProtocol* proto, const TBusServerSessionConfig& config, const TString& name) { - TBusServerSessionConfig patchedConfig = config; - patchedConfig.ExecuteOnMessageInWorkerPool = false; - if (!patchedConfig.Name) { - patchedConfig.Name = name; - } - if (!patchedConfig.Name) { - patchedConfig.Name = Impl->Name; - } - TBusServerSessionPtr session = - TBusServerSession::Create(proto, Impl->ModuleServerHandler.Get(), patchedConfig, &queue); - Impl->ServerSessions.push_back(session); - return session; - } - - TBusClientSessionPtr TBusModule::CreateDefaultSource( - TBusMessageQueue& queue, TBusProtocol* proto, const TBusClientSessionConfig& config, const TString& name) { - TBusClientSessionConfig patchedConfig = config; - patchedConfig.ExecuteOnReplyInWorkerPool = false; - if (!patchedConfig.Name) { - patchedConfig.Name = name; - } - if (!patchedConfig.Name) { - patchedConfig.Name = Impl->Name; - } - TBusClientSessionPtr session = - TBusClientSession::Create(proto, Impl->ModuleClientHandler.Get(), patchedConfig, &queue); - Impl->ClientSessions.push_back(session); - return session; - } - - TBusStarter* TBusModule::CreateDefaultStarter(TBusMessageQueue&, const TBusSessionConfig& config) { - TBusStarter* session = new TBusStarter(this, config); - Impl->Starters.push_back(session); - return session; - } - - void TBusModule::OnClientConnectionEvent(const TClientConnectionEvent& event) { - Y_UNUSED(event); - } - - TString TBusModule::GetStatus(unsigned flags) { - TString strReturn = Sprintf("%s\n", Impl->Name); - strReturn += Impl->GetStatus(flags); - return strReturn; - } + bool TBusModule::CreatePrivateSessions(TBusMessageQueue* queue) { + Impl->Queue = queue; + return true; + } + + int TBusModule::GetModuleSessionInFlight() const { + return Impl->Size(); + } + + TIntrusivePtr<TBusModuleInternal> TBusModule::GetInternal() { + return Impl.Get(); + } + + TBusServerSessionPtr TBusModule::CreateDefaultDestination( + TBusMessageQueue& queue, TBusProtocol* proto, const TBusServerSessionConfig& config, const TString& name) { + TBusServerSessionConfig patchedConfig = config; + patchedConfig.ExecuteOnMessageInWorkerPool = false; + if (!patchedConfig.Name) { + patchedConfig.Name = name; + } + if (!patchedConfig.Name) { + patchedConfig.Name = Impl->Name; + } + TBusServerSessionPtr session = + TBusServerSession::Create(proto, Impl->ModuleServerHandler.Get(), patchedConfig, &queue); + Impl->ServerSessions.push_back(session); + return session; + } + + TBusClientSessionPtr TBusModule::CreateDefaultSource( + TBusMessageQueue& queue, TBusProtocol* proto, const TBusClientSessionConfig& config, const TString& name) { + TBusClientSessionConfig patchedConfig = config; + patchedConfig.ExecuteOnReplyInWorkerPool = false; + if (!patchedConfig.Name) { + patchedConfig.Name = name; + } + if (!patchedConfig.Name) { + patchedConfig.Name = Impl->Name; + } + TBusClientSessionPtr session = + TBusClientSession::Create(proto, Impl->ModuleClientHandler.Get(), patchedConfig, &queue); + Impl->ClientSessions.push_back(session); + return session; + } + + TBusStarter* TBusModule::CreateDefaultStarter(TBusMessageQueue&, const TBusSessionConfig& config) { + TBusStarter* session = new TBusStarter(this, config); + Impl->Starters.push_back(session); + return session; + } + + void TBusModule::OnClientConnectionEvent(const TClientConnectionEvent& event) { + Y_UNUSED(event); + } + + TString TBusModule::GetStatus(unsigned flags) { + TString strReturn = Sprintf("%s\n", Impl->Name); + strReturn += Impl->GetStatus(flags); + return strReturn; + } } -void TBusModuleImpl::AddJob(TJobRunner* jobRunner) { +void TBusModuleImpl::AddJob(TJobRunner* jobRunner) { TWhatThreadDoesAcquireGuard<TMutex> G(Lock, "modules: acquiring lock for AddJob"); Jobs.push_back(jobRunner); jobRunner->JobStorageIterator = Jobs.end(); --jobRunner->JobStorageIterator; } -void TBusModuleImpl::DestroyJob(TJobRunner* job) { +void TBusModuleImpl::DestroyJob(TJobRunner* job) { Y_ASSERT(job->JobStorageIterator != TList<TJobRunner*>::iterator()); { @@ -837,7 +837,7 @@ EMessageStatus TBusModule::StartJob(TAutoPtr<TBusMessage> message) { Y_VERIFY(Impl->State == TBusModuleImpl::RUNNING); Y_VERIFY(!!Impl->Queue); - if ((unsigned)AtomicGet(Impl->JobCount) >= Impl->ModuleConfig.StarterMaxInFlight) { + if ((unsigned)AtomicGet(Impl->JobCount) >= Impl->ModuleConfig.StarterMaxInFlight) { return MESSAGE_BUSY; } diff --git a/library/cpp/messagebus/oldmodule/module.h b/library/cpp/messagebus/oldmodule/module.h index 8d1c4a5d52..c917f9cae7 100644 --- a/library/cpp/messagebus/oldmodule/module.h +++ b/library/cpp/messagebus/oldmodule/module.h @@ -1,37 +1,37 @@ -#pragma once - +#pragma once + /////////////////////////////////////////////////////////////////////////// /// \file /// \brief Application interface for modules - + /// NBus::TBusModule provides foundation for implementation of asynchnous /// modules that communicate with multiple external or local sessions -/// NBus::TBusSession. - +/// NBus::TBusSession. + /// To implement the module some virtual functions needs to be overridden: -/// NBus::TBusModule::CreateExtSession() creates and registers an +/// NBus::TBusModule::CreateExtSession() creates and registers an /// external session that receives incoming messages as input for module /// processing. - -/// When new incoming message arrives the new NBus::TBusJob is created. + +/// When new incoming message arrives the new NBus::TBusJob is created. /// NBus::TBusJob is somewhat similar to a thread, it maintains all the state -/// during processing of one incoming message. Default implementation of -/// NBus::TBusJob will maintain all send and received messages during +/// during processing of one incoming message. Default implementation of +/// NBus::TBusJob will maintain all send and received messages during /// lifetime of this job. Each message, status and reply can be found /// within NBus::TJobState using NBus::TBusJob::GetState(). If your module /// needs to maintain an additional information during lifetime of the job -/// you can derive your own class from NBus::TBusJob and override job +/// you can derive your own class from NBus::TBusJob and override job /// factory method NBus::IJobFactory::CreateJobInstance() to create your instances. - + /// Processing of a given message starts with a call to NBus::TBusModule::Start() /// handler that should be overridden in the module implementation. Within /// the callback handler module can perform any computation and access any /// datastore tables that it needs. The handler can also access any module -/// variables. However, same handler can be called from multiple threads so, +/// variables. However, same handler can be called from multiple threads so, /// it is recommended that handler only access read-only module level variables. - -/// Handler should use NBus::TBusJob::Send() to send messages to other client + +/// Handler should use NBus::TBusJob::Send() to send messages to other client /// sessions and it can use NBus::TBusJob::Reply() to send reply to the main /// job message. When handler is done, it returns the pointer to the next handler to call /// when all pending messages have cleared. If handler @@ -47,364 +47,364 @@ #include <util/generic/object_counter.h> namespace NBus { - class TBusJob; - class TBusModule; - - namespace NPrivate { - struct TCallJobHandlerWorkItem; - struct TBusModuleImpl; - struct TModuleServerHandler; - struct TModuleClientHandler; - struct TJobRunner; - } - - class TJobHandler { - protected: - typedef TJobHandler (TBusModule::*TBusHandlerPtr)(TBusJob* job, TBusMessage* mess); - TBusHandlerPtr MyPtr; - - public: - template <class B> - TJobHandler(TJobHandler (B::*fptr)(TBusJob* job, TBusMessage* mess)) { - MyPtr = static_cast<TBusHandlerPtr>(fptr); - } - TJobHandler(TBusHandlerPtr fptr = nullptr) { - MyPtr = fptr; - } + class TBusJob; + class TBusModule; + + namespace NPrivate { + struct TCallJobHandlerWorkItem; + struct TBusModuleImpl; + struct TModuleServerHandler; + struct TModuleClientHandler; + struct TJobRunner; + } + + class TJobHandler { + protected: + typedef TJobHandler (TBusModule::*TBusHandlerPtr)(TBusJob* job, TBusMessage* mess); + TBusHandlerPtr MyPtr; + + public: + template <class B> + TJobHandler(TJobHandler (B::*fptr)(TBusJob* job, TBusMessage* mess)) { + MyPtr = static_cast<TBusHandlerPtr>(fptr); + } + TJobHandler(TBusHandlerPtr fptr = nullptr) { + MyPtr = fptr; + } TJobHandler(const TJobHandler&) = default; TJobHandler& operator =(const TJobHandler&) = default; bool operator==(TJobHandler h) const { - return MyPtr == h.MyPtr; - } + return MyPtr == h.MyPtr; + } bool operator!=(TJobHandler h) const { - return MyPtr != h.MyPtr; - } - bool operator!() const { - return !MyPtr; - } - TJobHandler operator()(TBusModule* b, TBusJob* job, TBusMessage* mess) { - return (b->*MyPtr)(job, mess); - } - }; - - typedef void (TBusModule::*TReplyHandler)(TBusJob* job, EMessageStatus status, TBusMessage* mess, TBusMessage* reply); - - //////////////////////////////////////////////////// - /// \brief Pending message state - - struct TJobState { - friend class TBusJob; - friend class ::TCrawlerModule; - - TReplyHandler Handler; - EMessageStatus Status; - TBusMessage* Message; - TBusMessage* Reply; - TBusClientSession* Session; - size_t NumRetries; - size_t MaxRetries; - // If != NULL then use it as destination. - TNetAddr Addr; - bool UseAddr; - bool OneWay; - - private: - TJobState(TReplyHandler handler, - EMessageStatus status, - TBusMessage* mess, TBusClientSession* session, TBusMessage* reply, size_t maxRetries = 0, - const TNetAddr* addr = nullptr, bool oneWay = false) - : Handler(handler) - , Status(status) - , Message(mess) - , Reply(reply) - , Session(session) - , NumRetries(0) - , MaxRetries(maxRetries) - , OneWay(oneWay) - { - if (!!addr) { - Addr = *addr; - } - UseAddr = !!addr; - } - - public: - TString GetStatus(unsigned flags); - }; - - using TJobStateVec = TVector<TJobState>; - - ///////////////////////////////////////////////////////// - /// \brief Execution item = thread - - /// Maintains internal state of document in computation - class TBusJob { - TObjectCounter<TBusJob> ObjectCounter; - - private: - void CheckThreadCurrentJob(); - - public: - /// given a module and starter message - TBusJob(TBusModule* module, TBusMessage* message); - - /// destructor will free all the message that were send and received - virtual ~TBusJob(); - - TBusMessage* GetMessage() const { - return Message; - } - - TNetAddr GetPeerAddrNetAddr() const; - - /// send message to any other session or application - /// If addr is set then use it as destination. - void Send(TBusMessageAutoPtr mess, TBusClientSession* session, TReplyHandler rhandler, size_t maxRetries, const TNetAddr& addr); - void Send(TBusMessageAutoPtr mess, TBusClientSession* session, TReplyHandler rhandler = nullptr, size_t maxRetries = 0); - - void SendOneWayTo(TBusMessageAutoPtr req, TBusClientSession* session, const TNetAddr& addr); - void SendOneWayWithLocator(TBusMessageAutoPtr req, TBusClientSession* session); - - /// send reply to the starter message - virtual void SendReply(TBusMessageAutoPtr reply); - - /// set the flag to terminate job at the earliest convenience - void Cancel(EMessageStatus status); - - /// helper to put item on finished list of states - /// It should not be a part of public API, - /// so prohibit it for all except current users. - private: - friend class ::TCrawlerModule; - void PutState(const TJobState& state) { - Finished.push_back(state); + return MyPtr != h.MyPtr; + } + bool operator!() const { + return !MyPtr; + } + TJobHandler operator()(TBusModule* b, TBusJob* job, TBusMessage* mess) { + return (b->*MyPtr)(job, mess); + } + }; + + typedef void (TBusModule::*TReplyHandler)(TBusJob* job, EMessageStatus status, TBusMessage* mess, TBusMessage* reply); + + //////////////////////////////////////////////////// + /// \brief Pending message state + + struct TJobState { + friend class TBusJob; + friend class ::TCrawlerModule; + + TReplyHandler Handler; + EMessageStatus Status; + TBusMessage* Message; + TBusMessage* Reply; + TBusClientSession* Session; + size_t NumRetries; + size_t MaxRetries; + // If != NULL then use it as destination. + TNetAddr Addr; + bool UseAddr; + bool OneWay; + + private: + TJobState(TReplyHandler handler, + EMessageStatus status, + TBusMessage* mess, TBusClientSession* session, TBusMessage* reply, size_t maxRetries = 0, + const TNetAddr* addr = nullptr, bool oneWay = false) + : Handler(handler) + , Status(status) + , Message(mess) + , Reply(reply) + , Session(session) + , NumRetries(0) + , MaxRetries(maxRetries) + , OneWay(oneWay) + { + if (!!addr) { + Addr = *addr; + } + UseAddr = !!addr; } - public: - /// retrieve all pending messages - void GetPending(TJobStateVec* stateVec) { - Y_ASSERT(stateVec); - *stateVec = Pending; - } - - /// helper function to find state of previously sent messages - template <class MessageType> - TJobState* GetState(int* startFrom = nullptr) { - for (int i = startFrom ? *startFrom : 0; i < int(Finished.size()); i++) { - TJobState* call = &Finished[i]; - if (call->Reply != nullptr && dynamic_cast<MessageType*>(call->Reply)) { - if (startFrom) { - *startFrom = i; - } - return call; - } - if (call->Message != nullptr && dynamic_cast<MessageType*>(call->Message)) { - if (startFrom) { - *startFrom = i; - } - return call; - } + public: + TString GetStatus(unsigned flags); + }; + + using TJobStateVec = TVector<TJobState>; + + ///////////////////////////////////////////////////////// + /// \brief Execution item = thread + + /// Maintains internal state of document in computation + class TBusJob { + TObjectCounter<TBusJob> ObjectCounter; + + private: + void CheckThreadCurrentJob(); + + public: + /// given a module and starter message + TBusJob(TBusModule* module, TBusMessage* message); + + /// destructor will free all the message that were send and received + virtual ~TBusJob(); + + TBusMessage* GetMessage() const { + return Message; + } + + TNetAddr GetPeerAddrNetAddr() const; + + /// send message to any other session or application + /// If addr is set then use it as destination. + void Send(TBusMessageAutoPtr mess, TBusClientSession* session, TReplyHandler rhandler, size_t maxRetries, const TNetAddr& addr); + void Send(TBusMessageAutoPtr mess, TBusClientSession* session, TReplyHandler rhandler = nullptr, size_t maxRetries = 0); + + void SendOneWayTo(TBusMessageAutoPtr req, TBusClientSession* session, const TNetAddr& addr); + void SendOneWayWithLocator(TBusMessageAutoPtr req, TBusClientSession* session); + + /// send reply to the starter message + virtual void SendReply(TBusMessageAutoPtr reply); + + /// set the flag to terminate job at the earliest convenience + void Cancel(EMessageStatus status); + + /// helper to put item on finished list of states + /// It should not be a part of public API, + /// so prohibit it for all except current users. + private: + friend class ::TCrawlerModule; + void PutState(const TJobState& state) { + Finished.push_back(state); + } + + public: + /// retrieve all pending messages + void GetPending(TJobStateVec* stateVec) { + Y_ASSERT(stateVec); + *stateVec = Pending; + } + + /// helper function to find state of previously sent messages + template <class MessageType> + TJobState* GetState(int* startFrom = nullptr) { + for (int i = startFrom ? *startFrom : 0; i < int(Finished.size()); i++) { + TJobState* call = &Finished[i]; + if (call->Reply != nullptr && dynamic_cast<MessageType*>(call->Reply)) { + if (startFrom) { + *startFrom = i; + } + return call; + } + if (call->Message != nullptr && dynamic_cast<MessageType*>(call->Message)) { + if (startFrom) { + *startFrom = i; + } + return call; + } } - return nullptr; + return nullptr; } - /// helper function to find response for previously sent messages - template <class MessageType> - MessageType* Get(int* startFrom = nullptr) { - for (int i = startFrom ? *startFrom : 0; i < int(Finished.size()); i++) { - TJobState& call = Finished[i]; - if (call.Reply != nullptr && dynamic_cast<MessageType*>(call.Reply)) { - if (startFrom) { - *startFrom = i; - } - return static_cast<MessageType*>(call.Reply); - } - if (call.Message != nullptr && dynamic_cast<MessageType*>(call.Message)) { - if (startFrom) { - *startFrom = i; - } - return static_cast<MessageType*>(call.Message); - } + /// helper function to find response for previously sent messages + template <class MessageType> + MessageType* Get(int* startFrom = nullptr) { + for (int i = startFrom ? *startFrom : 0; i < int(Finished.size()); i++) { + TJobState& call = Finished[i]; + if (call.Reply != nullptr && dynamic_cast<MessageType*>(call.Reply)) { + if (startFrom) { + *startFrom = i; + } + return static_cast<MessageType*>(call.Reply); + } + if (call.Message != nullptr && dynamic_cast<MessageType*>(call.Message)) { + if (startFrom) { + *startFrom = i; + } + return static_cast<MessageType*>(call.Message); + } } - return nullptr; + return nullptr; } - /// helper function to find status for previously sent message - template <class MessageType> - EMessageStatus GetStatus(int* startFrom = nullptr) { - for (int i = startFrom ? *startFrom : 0; i < int(Finished.size()); i++) { - TJobState& call = Finished[i]; - if (call.Message != nullptr && dynamic_cast<MessageType*>(call.Message)) { - if (startFrom) { - *startFrom = i; - } - return call.Status; - } + /// helper function to find status for previously sent message + template <class MessageType> + EMessageStatus GetStatus(int* startFrom = nullptr) { + for (int i = startFrom ? *startFrom : 0; i < int(Finished.size()); i++) { + TJobState& call = Finished[i]; + if (call.Message != nullptr && dynamic_cast<MessageType*>(call.Message)) { + if (startFrom) { + *startFrom = i; + } + return call.Status; + } } - return MESSAGE_UNKNOWN; + return MESSAGE_UNKNOWN; } - /// helper function to clear state of previosly sent messages - template <class MessageType> - void Clear() { - for (size_t i = 0; i < Finished.size();) { - // `Finished.size() - i` decreases with each iteration - // we either increment i, or remove element from Finished. - TJobState& call = Finished[i]; - if (call.Message != nullptr && dynamic_cast<MessageType*>(call.Message)) { - ClearState(call); - } else { - ++i; - } + /// helper function to clear state of previosly sent messages + template <class MessageType> + void Clear() { + for (size_t i = 0; i < Finished.size();) { + // `Finished.size() - i` decreases with each iteration + // we either increment i, or remove element from Finished. + TJobState& call = Finished[i]; + if (call.Message != nullptr && dynamic_cast<MessageType*>(call.Message)) { + ClearState(call); + } else { + ++i; + } } } - /// helper function to clear state in order to try again - void ClearState(TJobState& state); - - /// clears all message states - void ClearAllMessageStates(); - - /// returns true if job is done - bool IsDone(); - - /// return human reabable status of this job - virtual TString GetStatus(unsigned flags); - - /// set sleep time for job - void Sleep(int milliSeconds); - - void CallJobHandlerOnly(); - - private: - bool CallJobHandler(); - void DoCallReplyHandler(TJobState&); - /// send out all Pending jobs, failed sends will be migrated to Finished - bool SendPending(); - bool AnyPendingToSend(); - - public: - /// helper to call from OnReply() and OnError() - int CallReplyHandler(EMessageStatus status, TBusMessage* mess, TBusMessage* reply); - - public: - TJobHandler Handler; ///< job handler to be executed within next CallJobHandler() - EMessageStatus Status; ///< set != MESSAGE_OK if job should terminate asap - private: - NPrivate::TJobRunner* Runner; - TBusMessage* Message; - THolder<TBusMessage> MessageHolder; - TOnMessageContext OnMessageContext; // starter - public: - bool ReplySent; - - private: - friend class TBusModule; - friend struct NPrivate::TBusModuleImpl; - friend struct NPrivate::TCallJobHandlerWorkItem; - friend struct NPrivate::TModuleServerHandler; - friend struct NPrivate::TModuleClientHandler; - friend struct NPrivate::TJobRunner; - - TJobStateVec Pending; ///< messages currently outstanding via Send() - TJobStateVec Finished; ///< messages that were replied to - TBusModule* Module; - NPrivate::TBusModuleImpl* ModuleImpl; ///< module which created the job - TBusInstant SleepUntil; ///< time to wakeup, 0 if no sleep + /// helper function to clear state in order to try again + void ClearState(TJobState& state); + + /// clears all message states + void ClearAllMessageStates(); + + /// returns true if job is done + bool IsDone(); + + /// return human reabable status of this job + virtual TString GetStatus(unsigned flags); + + /// set sleep time for job + void Sleep(int milliSeconds); + + void CallJobHandlerOnly(); + + private: + bool CallJobHandler(); + void DoCallReplyHandler(TJobState&); + /// send out all Pending jobs, failed sends will be migrated to Finished + bool SendPending(); + bool AnyPendingToSend(); + + public: + /// helper to call from OnReply() and OnError() + int CallReplyHandler(EMessageStatus status, TBusMessage* mess, TBusMessage* reply); + + public: + TJobHandler Handler; ///< job handler to be executed within next CallJobHandler() + EMessageStatus Status; ///< set != MESSAGE_OK if job should terminate asap + private: + NPrivate::TJobRunner* Runner; + TBusMessage* Message; + THolder<TBusMessage> MessageHolder; + TOnMessageContext OnMessageContext; // starter + public: + bool ReplySent; + + private: + friend class TBusModule; + friend struct NPrivate::TBusModuleImpl; + friend struct NPrivate::TCallJobHandlerWorkItem; + friend struct NPrivate::TModuleServerHandler; + friend struct NPrivate::TModuleClientHandler; + friend struct NPrivate::TJobRunner; + + TJobStateVec Pending; ///< messages currently outstanding via Send() + TJobStateVec Finished; ///< messages that were replied to + TBusModule* Module; + NPrivate::TBusModuleImpl* ModuleImpl; ///< module which created the job + TBusInstant SleepUntil; ///< time to wakeup, 0 if no sleep + }; + + //////////////////////////////////////////////////////////////////// + /// \brief Classes to implement basic module functionality + + class IJobFactory { + protected: + virtual ~IJobFactory() { + } + + public: + /// job factory method, override to create custom jobs + virtual TBusJob* CreateJobInstance(TBusMessage* message) = 0; }; - //////////////////////////////////////////////////////////////////// - /// \brief Classes to implement basic module functionality + struct TBusModuleConfig { + unsigned StarterMaxInFlight; - class IJobFactory { - protected: - virtual ~IJobFactory() { - } + struct TSecret { + TDuration SchedulePeriod; - public: - /// job factory method, override to create custom jobs - virtual TBusJob* CreateJobInstance(TBusMessage* message) = 0; - }; - - struct TBusModuleConfig { - unsigned StarterMaxInFlight; - - struct TSecret { - TDuration SchedulePeriod; + TSecret(); + }; + TSecret Secret; - TSecret(); - }; - TSecret Secret; + TBusModuleConfig(); + }; - TBusModuleConfig(); - }; - - namespace NPrivate { - struct TBusModuleInternal: public TAtomicRefCount<TBusModuleInternal> { - virtual TVector<TBusClientSessionPtr> GetClientSessionsInternal() = 0; - virtual TVector<TBusServerSessionPtr> GetServerSessionsInternal() = 0; - virtual TBusMessageQueue* GetQueue() = 0; + namespace NPrivate { + struct TBusModuleInternal: public TAtomicRefCount<TBusModuleInternal> { + virtual TVector<TBusClientSessionPtr> GetClientSessionsInternal() = 0; + virtual TVector<TBusServerSessionPtr> GetServerSessionsInternal() = 0; + virtual TBusMessageQueue* GetQueue() = 0; - virtual TString GetNameInternal() = 0; + virtual TString GetNameInternal() = 0; - virtual TString GetStatusSingleLine() = 0; + virtual TString GetStatusSingleLine() = 0; - virtual ~TBusModuleInternal() { - } - }; - } + virtual ~TBusModuleInternal() { + } + }; + } - class TBusModule: public IJobFactory, TNonCopyable { - friend class TBusJob; + class TBusModule: public IJobFactory, TNonCopyable { + friend class TBusJob; - TObjectCounter<TBusModule> ObjectCounter; + TObjectCounter<TBusModule> ObjectCounter; - TIntrusivePtr<NPrivate::TBusModuleImpl> Impl; + TIntrusivePtr<NPrivate::TBusModuleImpl> Impl; - public: - /// Each module should have a name which is used as protocol service - TBusModule(const char* name); - ~TBusModule() override; + public: + /// Each module should have a name which is used as protocol service + TBusModule(const char* name); + ~TBusModule() override; - const char* GetName() const; + const char* GetName() const; - void SetConfig(const TBusModuleConfig& config); + void SetConfig(const TBusModuleConfig& config); - /// get status of all jobs in flight - TString GetStatus(unsigned flags = 0); + /// get status of all jobs in flight + TString GetStatus(unsigned flags = 0); - /// called when application is about to start - virtual bool StartInput(); - /// called when application is about to exit - virtual bool Shutdown(); + /// called when application is about to start + virtual bool StartInput(); + /// called when application is about to exit + virtual bool Shutdown(); - // this default implementation just creates TBusJob object - TBusJob* CreateJobInstance(TBusMessage* message) override; + // this default implementation just creates TBusJob object + TBusJob* CreateJobInstance(TBusMessage* message) override; - EMessageStatus StartJob(TAutoPtr<TBusMessage> message); + EMessageStatus StartJob(TAutoPtr<TBusMessage> message); - /// creates private sessions, calls CreateExtSession(), should be called before StartInput() - bool CreatePrivateSessions(TBusMessageQueue* queue); + /// creates private sessions, calls CreateExtSession(), should be called before StartInput() + bool CreatePrivateSessions(TBusMessageQueue* queue); - virtual void OnClientConnectionEvent(const TClientConnectionEvent& event); + virtual void OnClientConnectionEvent(const TClientConnectionEvent& event); - public: - /// entry point into module, first function to call - virtual TJobHandler Start(TBusJob* job, TBusMessage* mess) = 0; - - protected: - /// override this function to create destination session - virtual TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) = 0; - - public: - int GetModuleSessionInFlight() const; - - TIntrusivePtr<NPrivate::TBusModuleInternal> GetInternal(); - - protected: - TBusServerSessionPtr CreateDefaultDestination(TBusMessageQueue& queue, TBusProtocol* proto, const TBusServerSessionConfig& config, const TString& name = TString()); - TBusClientSessionPtr CreateDefaultSource(TBusMessageQueue& queue, TBusProtocol* proto, const TBusClientSessionConfig& config, const TString& name = TString()); - TBusStarter* CreateDefaultStarter(TBusMessageQueue& unused, const TBusSessionConfig& config); - }; + public: + /// entry point into module, first function to call + virtual TJobHandler Start(TBusJob* job, TBusMessage* mess) = 0; + protected: + /// override this function to create destination session + virtual TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) = 0; + + public: + int GetModuleSessionInFlight() const; + + TIntrusivePtr<NPrivate::TBusModuleInternal> GetInternal(); + + protected: + TBusServerSessionPtr CreateDefaultDestination(TBusMessageQueue& queue, TBusProtocol* proto, const TBusServerSessionConfig& config, const TString& name = TString()); + TBusClientSessionPtr CreateDefaultSource(TBusMessageQueue& queue, TBusProtocol* proto, const TBusClientSessionConfig& config, const TString& name = TString()); + TBusStarter* CreateDefaultStarter(TBusMessageQueue& unused, const TBusSessionConfig& config); + }; + } diff --git a/library/cpp/messagebus/oldmodule/startsession.cpp b/library/cpp/messagebus/oldmodule/startsession.cpp index 7c38801d62..76171ba1d5 100644 --- a/library/cpp/messagebus/oldmodule/startsession.cpp +++ b/library/cpp/messagebus/oldmodule/startsession.cpp @@ -1,14 +1,14 @@ /////////////////////////////////////////////////////////// -/// \file +/// \file /// \brief Starter session implementation - + /// Starter session will generate emtpy message to insert /// into local session that are registered under same protocol - + /// Starter (will one day) automatically adjust number /// of message inflight to make sure that at least one of source /// sessions within message queue is at the limit (bottle neck) - + /// Maximum number of messages that starter will instert into /// the pipeline is configured by NBus::TBusSessionConfig::MaxInFlight @@ -19,46 +19,46 @@ #include <library/cpp/messagebus/ybus.h> namespace NBus { - void* TBusStarter::_starter(void* data) { - TBusStarter* pThis = static_cast<TBusStarter*>(data); - pThis->Starter(); - return nullptr; - } + void* TBusStarter::_starter(void* data) { + TBusStarter* pThis = static_cast<TBusStarter*>(data); + pThis->Starter(); + return nullptr; + } - TBusStarter::TBusStarter(TBusModule* module, const TBusSessionConfig& config) - : Module(module) - , Config(config) - , StartThread(_starter, this) - , Exiting(false) - { - StartThread.Start(); - } + TBusStarter::TBusStarter(TBusModule* module, const TBusSessionConfig& config) + : Module(module) + , Config(config) + , StartThread(_starter, this) + , Exiting(false) + { + StartThread.Start(); + } - TBusStarter::~TBusStarter() { - Shutdown(); - } + TBusStarter::~TBusStarter() { + Shutdown(); + } - void TBusStarter::Shutdown() { - { - TGuard<TMutex> g(ExitLock); - Exiting = true; - ExitSignal.Signal(); - } - StartThread.Join(); - } + void TBusStarter::Shutdown() { + { + TGuard<TMutex> g(ExitLock); + Exiting = true; + ExitSignal.Signal(); + } + StartThread.Join(); + } - void TBusStarter::Starter() { + void TBusStarter::Starter() { TGuard<TMutex> g(ExitLock); - while (!Exiting) { - TAutoPtr<TBusMessage> empty(new TBusMessage(0)); + while (!Exiting) { + TAutoPtr<TBusMessage> empty(new TBusMessage(0)); - EMessageStatus status = Module->StartJob(empty); + EMessageStatus status = Module->StartJob(empty); - if (Config.SendTimeout > 0) { - ExitSignal.WaitT(ExitLock, TDuration::MilliSeconds(Config.SendTimeout)); - } else { - ExitSignal.WaitT(ExitLock, (status == MESSAGE_BUSY) ? TDuration::MilliSeconds(1) : TDuration::Zero()); - } + if (Config.SendTimeout > 0) { + ExitSignal.WaitT(ExitLock, TDuration::MilliSeconds(Config.SendTimeout)); + } else { + ExitSignal.WaitT(ExitLock, (status == MESSAGE_BUSY) ? TDuration::MilliSeconds(1) : TDuration::Zero()); + } } } diff --git a/library/cpp/messagebus/oldmodule/startsession.h b/library/cpp/messagebus/oldmodule/startsession.h index 5e26e7e1e5..afe25ac809 100644 --- a/library/cpp/messagebus/oldmodule/startsession.h +++ b/library/cpp/messagebus/oldmodule/startsession.h @@ -5,30 +5,30 @@ #include <util/system/thread.h> namespace NBus { - class TBusModule; - - class TBusStarter { - private: - TBusModule* Module; - TBusSessionConfig Config; - TThread StartThread; - bool Exiting; - TCondVar ExitSignal; - TMutex ExitLock; - - static void* _starter(void* data); - - void Starter(); - - TString GetStatus(ui16 /*flags=YBUS_STATUS_CONNS*/) { - return ""; - } - - public: - TBusStarter(TBusModule* module, const TBusSessionConfig& config); - ~TBusStarter(); - - void Shutdown(); - }; - -} + class TBusModule; + + class TBusStarter { + private: + TBusModule* Module; + TBusSessionConfig Config; + TThread StartThread; + bool Exiting; + TCondVar ExitSignal; + TMutex ExitLock; + + static void* _starter(void* data); + + void Starter(); + + TString GetStatus(ui16 /*flags=YBUS_STATUS_CONNS*/) { + return ""; + } + + public: + TBusStarter(TBusModule* module, const TBusSessionConfig& config); + ~TBusStarter(); + + void Shutdown(); + }; + +} diff --git a/library/cpp/messagebus/protobuf/ybusbuf.cpp b/library/cpp/messagebus/protobuf/ybusbuf.cpp index 63415b3737..4848d2c323 100644 --- a/library/cpp/messagebus/protobuf/ybusbuf.cpp +++ b/library/cpp/messagebus/protobuf/ybusbuf.cpp @@ -41,10 +41,10 @@ TArrayRef<TBusBufferBase* const> TBusBufferProtocol::GetTypes() const { return Types; } -void TBusBufferProtocol::Serialize(const TBusMessage* mess, TBuffer& data) { +void TBusBufferProtocol::Serialize(const TBusMessage* mess, TBuffer& data) { TWhatThreadDoesPushPop pp("serialize protobuf message"); - const TBusHeader* header = mess->GetHeader(); + const TBusHeader* header = mess->GetHeader(); if (!IsRegisteredType(header->Type)) { Y_FAIL("unknown message type: %d", int(header->Type)); @@ -57,7 +57,7 @@ void TBusBufferProtocol::Serialize(const TBusMessage* mess, TBuffer& data) { unsigned size = bmess->GetRecord()->ByteSize(); data.Reserve(data.Size() + size); - char* after = (char*)bmess->GetRecord()->SerializeWithCachedSizesToArray((ui8*)data.Pos()); + char* after = (char*)bmess->GetRecord()->SerializeWithCachedSizesToArray((ui8*)data.Pos()); Y_VERIFY(after - data.Pos() == size); data.Advance(size); @@ -66,7 +66,7 @@ void TBusBufferProtocol::Serialize(const TBusMessage* mess, TBuffer& data) { TAutoPtr<TBusMessage> TBusBufferProtocol::Deserialize(ui16 messageType, TArrayRef<const char> payload) { TWhatThreadDoesPushPop pp("deserialize protobuf message"); - TBusBufferBase* messageTemplate = FindType(messageType); + TBusBufferBase* messageTemplate = FindType(messageType); if (messageTemplate == nullptr) { return nullptr; //Y_FAIL("unknown message type: %d", unsigned(messageType)); diff --git a/library/cpp/messagebus/protobuf/ybusbuf.h b/library/cpp/messagebus/protobuf/ybusbuf.h index 57b4267ea5..b01338f986 100644 --- a/library/cpp/messagebus/protobuf/ybusbuf.h +++ b/library/cpp/messagebus/protobuf/ybusbuf.h @@ -12,222 +12,222 @@ #include <array> namespace NBus { - using TBusBufferRecord = ::google::protobuf::Message; - - template <class TBufferMessage> - class TBusBufferMessagePtr; - template <class TBufferMessage> - class TBusBufferMessageAutoPtr; - - class TBusBufferBase: public TBusMessage { - public: - TBusBufferBase(int type) - : TBusMessage((ui16)type) - { - } - TBusBufferBase(ECreateUninitialized) - : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) - { - } - - ui16 GetType() const { - return GetHeader()->Type; - } - - virtual TBusBufferRecord* GetRecord() const = 0; - virtual TBusBufferBase* New() = 0; - }; - - /////////////////////////////////////////////////////////////////// - /// \brief Template for all messages that have protobuf description - - /// @param TBufferRecord is record described in .proto file with namespace - /// @param MessageFile is offset for .proto file message ids - - /// \attention If you want one protocol NBus::TBusBufferProtocol to handle - /// messageges described in different .proto files, make sure that they have - /// unique values for MessageFile - - template <class TBufferRecord, int MType> - class TBusBufferMessage: public TBusBufferBase { - public: - static const int MessageType = MType; - - typedef TBusBufferMessagePtr<TBusBufferMessage<TBufferRecord, MType>> TPtr; - typedef TBusBufferMessageAutoPtr<TBusBufferMessage<TBufferRecord, MType>> TAutoPtr; - - public: - typedef TBufferRecord RecordType; - TBufferRecord Record; - - public: - TBusBufferMessage() - : TBusBufferBase(MessageType) - { - } - TBusBufferMessage(ECreateUninitialized) - : TBusBufferBase(MESSAGE_CREATE_UNINITIALIZED) - { - } - explicit TBusBufferMessage(const TBufferRecord& record) - : TBusBufferBase(MessageType) - , Record(record) - { - } - explicit TBusBufferMessage(TBufferRecord&& record) - : TBusBufferBase(MessageType) - , Record(std::move(record)) - { - } - - public: - TBusBufferRecord* GetRecord() const override { - return (TBusBufferRecord*)&Record; - } - TBusBufferBase* New() override { - return new TBusBufferMessage<TBufferRecord, MessageType>(); - } - }; - - template <class TSelf, class TBufferMessage> - class TBusBufferMessagePtrBase { - public: - typedef typename TBufferMessage::RecordType RecordType; - - private: - TSelf* GetSelf() { - return static_cast<TSelf*>(this); - } - const TSelf* GetSelf() const { - return static_cast<const TSelf*>(this); - } - - public: - RecordType* operator->() { - Y_ASSERT(GetSelf()->Get()); - return &(GetSelf()->Get()->Record); - } - const RecordType* operator->() const { - Y_ASSERT(GetSelf()->Get()); - return &(GetSelf()->Get()->Record); - } - RecordType& operator*() { - Y_ASSERT(GetSelf()->Get()); - return GetSelf()->Get()->Record; - } - const RecordType& operator*() const { - Y_ASSERT(GetSelf()->Get()); - return GetSelf()->Get()->Record; - } - - TBusHeader* GetHeader() { - return GetSelf()->Get()->GetHeader(); - } - const TBusHeader* GetHeader() const { - return GetSelf()->Get()->GetHeader(); - } - }; - - template <class TBufferMessage> - class TBusBufferMessagePtr: public TBusBufferMessagePtrBase<TBusBufferMessagePtr<TBufferMessage>, TBufferMessage> { - protected: - TBufferMessage* Holder; - - public: - TBusBufferMessagePtr(TBufferMessage* mess) - : Holder(mess) - { - } - static TBusBufferMessagePtr<TBufferMessage> DynamicCast(TBusMessage* message) { - return dynamic_cast<TBufferMessage*>(message); - } - TBufferMessage* Get() { - return Holder; - } - const TBufferMessage* Get() const { - return Holder; - } - - operator TBufferMessage*() { - return Holder; - } - operator const TBufferMessage*() const { - return Holder; - } - - operator TAutoPtr<TBusMessage>() { - TAutoPtr<TBusMessage> r(Holder); - Holder = 0; - return r; - } - operator TBusMessageAutoPtr() { - TBusMessageAutoPtr r(Holder); - Holder = nullptr; - return r; - } - }; - - template <class TBufferMessage> - class TBusBufferMessageAutoPtr: public TBusBufferMessagePtrBase<TBusBufferMessageAutoPtr<TBufferMessage>, TBufferMessage> { - public: - TAutoPtr<TBufferMessage> AutoPtr; - - public: - TBusBufferMessageAutoPtr() { - } - TBusBufferMessageAutoPtr(TBufferMessage* message) - : AutoPtr(message) - { - } - - TBufferMessage* Get() { - return AutoPtr.Get(); - } - const TBufferMessage* Get() const { - return AutoPtr.Get(); - } - - TBufferMessage* Release() const { - return AutoPtr.Release(); - } - - operator TAutoPtr<TBusMessage>() { - return AutoPtr.Release(); - } - operator TBusMessageAutoPtr() { - return AutoPtr.Release(); - } - }; - - ///////////////////////////////////////////// - /// \brief Generic protocol object for messages descibed with protobuf - - /// \attention If you mix messages in the same protocol from more than - /// .proto file make sure that they have different MessageFile parameter - /// in the NBus::TBusBufferMessage template - - class TBusBufferProtocol: public TBusProtocol { - private: - TVector<TBusBufferBase*> Types; - std::array<ui32, ((1 << 16) >> 5)> TypeMask; - - TBusBufferBase* FindType(int type); - bool IsRegisteredType(unsigned type); - - public: - TBusBufferProtocol(TBusService name, int port); - - ~TBusBufferProtocol() override; - - /// register all the message that this protocol should handle - void RegisterType(TAutoPtr<TBusBufferBase> mess); - - TArrayRef<TBusBufferBase* const> GetTypes() const; - - /// serialized protocol specific data into TBusData - void Serialize(const TBusMessage* mess, TBuffer& data) override; - - TAutoPtr<TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) override; - }; - -} + using TBusBufferRecord = ::google::protobuf::Message; + + template <class TBufferMessage> + class TBusBufferMessagePtr; + template <class TBufferMessage> + class TBusBufferMessageAutoPtr; + + class TBusBufferBase: public TBusMessage { + public: + TBusBufferBase(int type) + : TBusMessage((ui16)type) + { + } + TBusBufferBase(ECreateUninitialized) + : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) + { + } + + ui16 GetType() const { + return GetHeader()->Type; + } + + virtual TBusBufferRecord* GetRecord() const = 0; + virtual TBusBufferBase* New() = 0; + }; + + /////////////////////////////////////////////////////////////////// + /// \brief Template for all messages that have protobuf description + + /// @param TBufferRecord is record described in .proto file with namespace + /// @param MessageFile is offset for .proto file message ids + + /// \attention If you want one protocol NBus::TBusBufferProtocol to handle + /// messageges described in different .proto files, make sure that they have + /// unique values for MessageFile + + template <class TBufferRecord, int MType> + class TBusBufferMessage: public TBusBufferBase { + public: + static const int MessageType = MType; + + typedef TBusBufferMessagePtr<TBusBufferMessage<TBufferRecord, MType>> TPtr; + typedef TBusBufferMessageAutoPtr<TBusBufferMessage<TBufferRecord, MType>> TAutoPtr; + + public: + typedef TBufferRecord RecordType; + TBufferRecord Record; + + public: + TBusBufferMessage() + : TBusBufferBase(MessageType) + { + } + TBusBufferMessage(ECreateUninitialized) + : TBusBufferBase(MESSAGE_CREATE_UNINITIALIZED) + { + } + explicit TBusBufferMessage(const TBufferRecord& record) + : TBusBufferBase(MessageType) + , Record(record) + { + } + explicit TBusBufferMessage(TBufferRecord&& record) + : TBusBufferBase(MessageType) + , Record(std::move(record)) + { + } + + public: + TBusBufferRecord* GetRecord() const override { + return (TBusBufferRecord*)&Record; + } + TBusBufferBase* New() override { + return new TBusBufferMessage<TBufferRecord, MessageType>(); + } + }; + + template <class TSelf, class TBufferMessage> + class TBusBufferMessagePtrBase { + public: + typedef typename TBufferMessage::RecordType RecordType; + + private: + TSelf* GetSelf() { + return static_cast<TSelf*>(this); + } + const TSelf* GetSelf() const { + return static_cast<const TSelf*>(this); + } + + public: + RecordType* operator->() { + Y_ASSERT(GetSelf()->Get()); + return &(GetSelf()->Get()->Record); + } + const RecordType* operator->() const { + Y_ASSERT(GetSelf()->Get()); + return &(GetSelf()->Get()->Record); + } + RecordType& operator*() { + Y_ASSERT(GetSelf()->Get()); + return GetSelf()->Get()->Record; + } + const RecordType& operator*() const { + Y_ASSERT(GetSelf()->Get()); + return GetSelf()->Get()->Record; + } + + TBusHeader* GetHeader() { + return GetSelf()->Get()->GetHeader(); + } + const TBusHeader* GetHeader() const { + return GetSelf()->Get()->GetHeader(); + } + }; + + template <class TBufferMessage> + class TBusBufferMessagePtr: public TBusBufferMessagePtrBase<TBusBufferMessagePtr<TBufferMessage>, TBufferMessage> { + protected: + TBufferMessage* Holder; + + public: + TBusBufferMessagePtr(TBufferMessage* mess) + : Holder(mess) + { + } + static TBusBufferMessagePtr<TBufferMessage> DynamicCast(TBusMessage* message) { + return dynamic_cast<TBufferMessage*>(message); + } + TBufferMessage* Get() { + return Holder; + } + const TBufferMessage* Get() const { + return Holder; + } + + operator TBufferMessage*() { + return Holder; + } + operator const TBufferMessage*() const { + return Holder; + } + + operator TAutoPtr<TBusMessage>() { + TAutoPtr<TBusMessage> r(Holder); + Holder = 0; + return r; + } + operator TBusMessageAutoPtr() { + TBusMessageAutoPtr r(Holder); + Holder = nullptr; + return r; + } + }; + + template <class TBufferMessage> + class TBusBufferMessageAutoPtr: public TBusBufferMessagePtrBase<TBusBufferMessageAutoPtr<TBufferMessage>, TBufferMessage> { + public: + TAutoPtr<TBufferMessage> AutoPtr; + + public: + TBusBufferMessageAutoPtr() { + } + TBusBufferMessageAutoPtr(TBufferMessage* message) + : AutoPtr(message) + { + } + + TBufferMessage* Get() { + return AutoPtr.Get(); + } + const TBufferMessage* Get() const { + return AutoPtr.Get(); + } + + TBufferMessage* Release() const { + return AutoPtr.Release(); + } + + operator TAutoPtr<TBusMessage>() { + return AutoPtr.Release(); + } + operator TBusMessageAutoPtr() { + return AutoPtr.Release(); + } + }; + + ///////////////////////////////////////////// + /// \brief Generic protocol object for messages descibed with protobuf + + /// \attention If you mix messages in the same protocol from more than + /// .proto file make sure that they have different MessageFile parameter + /// in the NBus::TBusBufferMessage template + + class TBusBufferProtocol: public TBusProtocol { + private: + TVector<TBusBufferBase*> Types; + std::array<ui32, ((1 << 16) >> 5)> TypeMask; + + TBusBufferBase* FindType(int type); + bool IsRegisteredType(unsigned type); + + public: + TBusBufferProtocol(TBusService name, int port); + + ~TBusBufferProtocol() override; + + /// register all the message that this protocol should handle + void RegisterType(TAutoPtr<TBusBufferBase> mess); + + TArrayRef<TBusBufferBase* const> GetTypes() const; + + /// serialized protocol specific data into TBusData + void Serialize(const TBusMessage* mess, TBuffer& data) override; + + TAutoPtr<TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) override; + }; + +} diff --git a/library/cpp/messagebus/queue_config.cpp b/library/cpp/messagebus/queue_config.cpp index 78fb52ee49..6e665ce8ce 100644 --- a/library/cpp/messagebus/queue_config.cpp +++ b/library/cpp/messagebus/queue_config.cpp @@ -8,11 +8,11 @@ TBusQueueConfig::TBusQueueConfig() { } void TBusQueueConfig::ConfigureLastGetopt( - NLastGetopt::TOpts& opts, const TString& prefix) { + NLastGetopt::TOpts& opts, const TString& prefix) { opts.AddLongOption(prefix + "worker-count") - .RequiredArgument("COUNT") - .DefaultValue(ToString(NumWorkers)) - .StoreResult(&NumWorkers); + .RequiredArgument("COUNT") + .DefaultValue(ToString(NumWorkers)) + .StoreResult(&NumWorkers); } TString TBusQueueConfig::PrintToString() const { diff --git a/library/cpp/messagebus/queue_config.h b/library/cpp/messagebus/queue_config.h index a9955f0c70..61a9395886 100644 --- a/library/cpp/messagebus/queue_config.h +++ b/library/cpp/messagebus/queue_config.h @@ -3,17 +3,17 @@ #include <library/cpp/getopt/last_getopt.h> namespace NBus { - ////////////////////////////////////////////////////////////////// - /// \brief Configuration for message queue - struct TBusQueueConfig { - TString Name; - int NumWorkers; ///< number of threads calling OnMessage(), OnReply() handlers + ////////////////////////////////////////////////////////////////// + /// \brief Configuration for message queue + struct TBusQueueConfig { + TString Name; + int NumWorkers; ///< number of threads calling OnMessage(), OnReply() handlers - TBusQueueConfig(); ///< initializes with default settings + TBusQueueConfig(); ///< initializes with default settings - void ConfigureLastGetopt(NLastGetopt::TOpts&, const TString& prefix = "mb-"); + void ConfigureLastGetopt(NLastGetopt::TOpts&, const TString& prefix = "mb-"); - TString PrintToString() const; - }; + TString PrintToString() const; + }; } diff --git a/library/cpp/messagebus/rain_check/core/coro.cpp b/library/cpp/messagebus/rain_check/core/coro.cpp index 500841dd5b..8dda6c9f97 100644 --- a/library/cpp/messagebus/rain_check/core/coro.cpp +++ b/library/cpp/messagebus/rain_check/core/coro.cpp @@ -26,12 +26,12 @@ TCoroTaskRunner::~TCoroTaskRunner() { Y_ASSERT(CoroDone); } -Y_POD_STATIC_THREAD(TContMachineContext*) -CallerContext; -Y_POD_STATIC_THREAD(TCoroTaskRunner*) -Task; +Y_POD_STATIC_THREAD(TContMachineContext*) +CallerContext; +Y_POD_STATIC_THREAD(TCoroTaskRunner*) +Task; -bool TCoroTaskRunner::ReplyReceived() { +bool TCoroTaskRunner::ReplyReceived() { Y_ASSERT(!CoroDone); TContMachineContext me; @@ -49,12 +49,12 @@ bool TCoroTaskRunner::ReplyReceived() { return !CoroDone; } -void NRainCheck::TCoroTaskRunner::DoRun() { +void NRainCheck::TCoroTaskRunner::DoRun() { GetImpl()->Run(); CoroDone = true; ContMachineContext.SwitchTo(CallerContext); } -void NRainCheck::ICoroTask::WaitForSubtasks() { +void NRainCheck::ICoroTask::WaitForSubtasks() { Task->ContMachineContext.SwitchTo(CallerContext); } diff --git a/library/cpp/messagebus/rain_check/core/coro.h b/library/cpp/messagebus/rain_check/core/coro.h index 95e2a30f9b..c62bef5ebc 100644 --- a/library/cpp/messagebus/rain_check/core/coro.h +++ b/library/cpp/messagebus/rain_check/core/coro.h @@ -14,7 +14,7 @@ namespace NRainCheck { class TCoroTaskRunner: public TTaskRunnerBase, private ITrampoLine { friend class ICoroTask; - + private: NPrivate::TCoroStack Stack; TContMachineContext ContMachineContext; @@ -31,9 +31,9 @@ namespace NRainCheck { void DoRun() override /* override */; - ICoroTask* GetImpl() { - return (ICoroTask*)GetImplBase(); - } + ICoroTask* GetImpl() { + return (ICoroTask*)GetImplBase(); + } }; class ICoroTask: public ITaskBase { @@ -46,10 +46,10 @@ namespace NRainCheck { typedef TCoroTaskRunner TTaskRunner; typedef ICoroTask ITask; - ICoroTask(size_t stackSize = 0x2000) - : StackSize(stackSize) - { - } + ICoroTask(size_t stackSize = 0x2000) + : StackSize(stackSize) + { + } virtual void Run() = 0; static void WaitForSubtasks(); diff --git a/library/cpp/messagebus/rain_check/core/coro_stack.cpp b/library/cpp/messagebus/rain_check/core/coro_stack.cpp index 83b984ca6e..be69939415 100644 --- a/library/cpp/messagebus/rain_check/core/coro_stack.cpp +++ b/library/cpp/messagebus/rain_check/core/coro_stack.cpp @@ -22,11 +22,11 @@ TCoroStack::TCoroStack(size_t size) *MagicNumberLocation() = MAGIC_NUMBER; #if defined(WITH_VALGRIND) - ValgrindStackId = VALGRIND_STACK_REGISTER(Data(), (char*)Data() + Size()); + ValgrindStackId = VALGRIND_STACK_REGISTER(Data(), (char*)Data() + Size()); #endif } -TCoroStack::~TCoroStack() { +TCoroStack::~TCoroStack() { #if defined(WITH_VALGRIND) VALGRIND_STACK_DEREGISTER(ValgrindStackId); #endif diff --git a/library/cpp/messagebus/rain_check/core/coro_stack.h b/library/cpp/messagebus/rain_check/core/coro_stack.h index 2f3520e6e4..6d40da1ea1 100644 --- a/library/cpp/messagebus/rain_check/core/coro_stack.h +++ b/library/cpp/messagebus/rain_check/core/coro_stack.h @@ -4,51 +4,51 @@ #include <util/generic/ptr.h> #include <util/system/valgrind.h> -namespace NRainCheck { - namespace NPrivate { - struct TCoroStack { - THolder<void, TFree> DataHolder; - size_t SizeValue; +namespace NRainCheck { + namespace NPrivate { + struct TCoroStack { + THolder<void, TFree> DataHolder; + size_t SizeValue; #if defined(WITH_VALGRIND) - size_t ValgrindStackId; + size_t ValgrindStackId; #endif - TCoroStack(size_t size); - ~TCoroStack(); + TCoroStack(size_t size); + ~TCoroStack(); - void* Data() { - return DataHolder.Get(); - } + void* Data() { + return DataHolder.Get(); + } - size_t Size() { - return SizeValue; - } + size_t Size() { + return SizeValue; + } TArrayRef<char> MemRegion() { return TArrayRef((char*)Data(), Size()); - } + } - ui32* MagicNumberLocation() { + ui32* MagicNumberLocation() { #if STACK_GROW_DOWN == 1 - return (ui32*)Data(); + return (ui32*)Data(); #elif STACK_GROW_DOWN == 0 - return ((ui32*)(((char*)Data()) + Size())) - 1; + return ((ui32*)(((char*)Data()) + Size())) - 1; #else #error "unknown" #endif - } + } - static void FailStackOverflow(); + static void FailStackOverflow(); - inline void VerifyNoStackOverflow() noexcept { - if (Y_UNLIKELY(*MagicNumberLocation() != MAGIC_NUMBER)) { - FailStackOverflow(); - } + inline void VerifyNoStackOverflow() noexcept { + if (Y_UNLIKELY(*MagicNumberLocation() != MAGIC_NUMBER)) { + FailStackOverflow(); + } } - static const ui32 MAGIC_NUMBER = 0xAB4D15FE; - }; + static const ui32 MAGIC_NUMBER = 0xAB4D15FE; + }; - } -} + } +} diff --git a/library/cpp/messagebus/rain_check/core/coro_ut.cpp b/library/cpp/messagebus/rain_check/core/coro_ut.cpp index 61a33584a5..95edce680b 100644 --- a/library/cpp/messagebus/rain_check/core/coro_ut.cpp +++ b/library/cpp/messagebus/rain_check/core/coro_ut.cpp @@ -8,7 +8,7 @@ using namespace NRainCheck; Y_UNIT_TEST_SUITE(RainCheckCoro) { - struct TSimpleCoroTask : ICoroTask { + struct TSimpleCoroTask : ICoroTask { TTestSync* const TestSync; TSimpleCoroTask(TTestEnv*, TTestSync* testSync) @@ -30,7 +30,7 @@ Y_UNIT_TEST_SUITE(RainCheckCoro) { testSync.WaitForAndIncrement(1); } - struct TSleepCoroTask : ICoroTask { + struct TSleepCoroTask : ICoroTask { TTestEnv* const Env; TTestSync* const TestSync; @@ -59,7 +59,7 @@ Y_UNIT_TEST_SUITE(RainCheckCoro) { testSync.WaitForAndIncrement(1); } - struct TSubtask : ICoroTask { + struct TSubtask : ICoroTask { TTestEnv* const Env; TTestSync* const TestSync; @@ -74,7 +74,7 @@ Y_UNIT_TEST_SUITE(RainCheckCoro) { } }; - struct TSpawnCoroTask : ICoroTask { + struct TSpawnCoroTask : ICoroTask { TTestEnv* const Env; TTestSync* const TestSync; diff --git a/library/cpp/messagebus/rain_check/core/env.h b/library/cpp/messagebus/rain_check/core/env.h index f6dd7fceb6..945364c3b3 100644 --- a/library/cpp/messagebus/rain_check/core/env.h +++ b/library/cpp/messagebus/rain_check/core/env.h @@ -10,15 +10,15 @@ namespace NRainCheck { struct IEnv { virtual ::NActor::TExecutor* GetExecutor() = 0; - virtual ~IEnv() { - } + virtual ~IEnv() { + } }; template <typename TSelf> struct TEnvTemplate: public IEnv { template <typename TTask, typename TParam> TIntrusivePtr<typename TTask::TTaskRunner> SpawnTask(TParam param) { - return ::NRainCheck::SpawnTask<TTask, TSelf>((TSelf*)this, param); + return ::NRainCheck::SpawnTask<TTask, TSelf>((TSelf*)this, param); } }; @@ -29,19 +29,19 @@ namespace NRainCheck { TSimpleEnvTemplate(unsigned threadCount = 0) : Executor(new ::NActor::TExecutor(threadCount != 0 ? threadCount : 4)) - { - } + { + } - ::NActor::TExecutor* GetExecutor() override { - return Executor.Get(); - } + ::NActor::TExecutor* GetExecutor() override { + return Executor.Get(); + } }; struct TSimpleEnv: public TSimpleEnvTemplate<TSimpleEnv> { - TSimpleEnv(unsigned threadCount = 0) - : TSimpleEnvTemplate<TSimpleEnv>(threadCount) - { - } + TSimpleEnv(unsigned threadCount = 0) + : TSimpleEnvTemplate<TSimpleEnv>(threadCount) + { + } }; } diff --git a/library/cpp/messagebus/rain_check/core/simple.cpp b/library/cpp/messagebus/rain_check/core/simple.cpp index 70182b2f93..46a1d0e8aa 100644 --- a/library/cpp/messagebus/rain_check/core/simple.cpp +++ b/library/cpp/messagebus/rain_check/core/simple.cpp @@ -12,7 +12,7 @@ TSimpleTaskRunner::~TSimpleTaskRunner() { Y_ASSERT(!ContinueFunc); } -bool TSimpleTaskRunner::ReplyReceived() { +bool TSimpleTaskRunner::ReplyReceived() { ContinueFunc = (GetImpl()->*(ContinueFunc.Func))(); return !!ContinueFunc; } diff --git a/library/cpp/messagebus/rain_check/core/simple.h b/library/cpp/messagebus/rain_check/core/simple.h index 20e1bf19f5..82664cabab 100644 --- a/library/cpp/messagebus/rain_check/core/simple.h +++ b/library/cpp/messagebus/rain_check/core/simple.h @@ -15,17 +15,17 @@ namespace NRainCheck { public: TContinueFunc() : Func(nullptr) - { - } + { + } TContinueFunc(void*) : Func(nullptr) - { - } + { + } template <typename TTask> TContinueFunc(TContinueFunc (TTask::*func)()) - : Func((TFunc)func) + : Func((TFunc)func) { static_assert((std::is_base_of<ISimpleTask, TTask>::value), "expect (std::is_base_of<ISimpleTask, TTask>::value)"); } @@ -46,9 +46,9 @@ namespace NRainCheck { bool ReplyReceived() override /* override */; - ISimpleTask* GetImpl() { - return (ISimpleTask*)GetImplBase(); - } + ISimpleTask* GetImpl() { + return (ISimpleTask*)GetImplBase(); + } }; class ISimpleTask: public ITaskBase { diff --git a/library/cpp/messagebus/rain_check/core/simple_ut.cpp b/library/cpp/messagebus/rain_check/core/simple_ut.cpp index d4545e05aa..820996d11a 100644 --- a/library/cpp/messagebus/rain_check/core/simple_ut.cpp +++ b/library/cpp/messagebus/rain_check/core/simple_ut.cpp @@ -16,8 +16,8 @@ Y_UNIT_TEST_SUITE(RainCheckSimple) { TTaskWithCompletionCallback(TTestEnv* env, TTestSync* testSync) : Env(env) , TestSync(testSync) - { - } + { + } TSubtaskCompletion SleepCompletion; @@ -38,7 +38,7 @@ Y_UNIT_TEST_SUITE(RainCheckSimple) { SleepCompletion.SetCompletionCallback(&TTaskWithCompletionCallback::NextSleepCompletionCallback); } - void NextSleepCompletionCallback(TSubtaskCompletion*) { + void NextSleepCompletionCallback(TSubtaskCompletion*) { TestSync->CheckAndIncrement(2); } diff --git a/library/cpp/messagebus/rain_check/core/sleep.cpp b/library/cpp/messagebus/rain_check/core/sleep.cpp index f5d0b4cac9..4fb645b7d6 100644 --- a/library/cpp/messagebus/rain_check/core/sleep.cpp +++ b/library/cpp/messagebus/rain_check/core/sleep.cpp @@ -18,7 +18,7 @@ NRainCheck::TSleepService::TSleepService() { } -NRainCheck::TSleepService::~TSleepService() { +NRainCheck::TSleepService::~TSleepService() { if (!!SchedulerHolder) { Scheduler->Stop(); } @@ -40,7 +40,7 @@ namespace { }; } -void TSleepService::Sleep(TSubtaskCompletion* r, TDuration duration) { +void TSleepService::Sleep(TSubtaskCompletion* r, TDuration duration) { TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); r->SetRunning(current); Scheduler->Schedule(new TSleepServiceScheduleItem(r, TInstant::Now() + duration)); diff --git a/library/cpp/messagebus/rain_check/core/sleep.h b/library/cpp/messagebus/rain_check/core/sleep.h index 1a7a1f8674..dc63c098df 100644 --- a/library/cpp/messagebus/rain_check/core/sleep.h +++ b/library/cpp/messagebus/rain_check/core/sleep.h @@ -11,7 +11,7 @@ namespace NRainCheck { private: THolder< ::NBus::NPrivate::TScheduler> SchedulerHolder; ::NBus::NPrivate::TScheduler* const Scheduler; - + public: TSleepService(::NBus::NPrivate::TScheduler*); TSleepService(); diff --git a/library/cpp/messagebus/rain_check/core/sleep_ut.cpp b/library/cpp/messagebus/rain_check/core/sleep_ut.cpp index 2ae85a87b1..cddcb2b44a 100644 --- a/library/cpp/messagebus/rain_check/core/sleep_ut.cpp +++ b/library/cpp/messagebus/rain_check/core/sleep_ut.cpp @@ -15,8 +15,8 @@ Y_UNIT_TEST_SUITE(Sleep) { TTestTask(TSimpleEnv* env, TTestSync* testSync) : Env(env) , TestSync(testSync) - { - } + { + } TSubtaskCompletion Sleep; diff --git a/library/cpp/messagebus/rain_check/core/spawn.cpp b/library/cpp/messagebus/rain_check/core/spawn.cpp index c570355fbe..879192e26a 100644 --- a/library/cpp/messagebus/rain_check/core/spawn.cpp +++ b/library/cpp/messagebus/rain_check/core/spawn.cpp @@ -1,5 +1,5 @@ #include "spawn.h" -void NRainCheck::NPrivate::SpawnTaskImpl(TTaskRunnerBase* task) { +void NRainCheck::NPrivate::SpawnTaskImpl(TTaskRunnerBase* task) { task->Schedule(); } diff --git a/library/cpp/messagebus/rain_check/core/spawn.h b/library/cpp/messagebus/rain_check/core/spawn.h index f2b146bf29..f61b8cc8c2 100644 --- a/library/cpp/messagebus/rain_check/core/spawn.h +++ b/library/cpp/messagebus/rain_check/core/spawn.h @@ -38,7 +38,7 @@ namespace NRainCheck { template <typename TTask, typename TEnv, typename TParam> TIntrusivePtr<typename TTask::TTaskRunner> SpawnTask(TEnv* env, TParam param1, ISubtaskListener* subtaskListener = &TNopSubtaskListener::Instance) { return NPrivate::SpawnTaskWithRunner< - TTask, typename TTask::ITask, typename TTask::TTaskRunner, TEnv, TParam>(env, param1, subtaskListener); + TTask, typename TTask::ITask, typename TTask::TTaskRunner, TEnv, TParam>(env, param1, subtaskListener); } // Instantiate and start subtask of given task. diff --git a/library/cpp/messagebus/rain_check/core/spawn_ut.cpp b/library/cpp/messagebus/rain_check/core/spawn_ut.cpp index ba5a5e41cf..a0391953e9 100644 --- a/library/cpp/messagebus/rain_check/core/spawn_ut.cpp +++ b/library/cpp/messagebus/rain_check/core/spawn_ut.cpp @@ -19,8 +19,8 @@ Y_UNIT_TEST_SUITE(Spawn) { TTestTask(TSimpleEnv*, TTestSync* testSync) : TestSync(testSync) , I(0) - { - } + { + } TSystemEvent Started; @@ -60,8 +60,8 @@ Y_UNIT_TEST_SUITE(Spawn) { TSubtask(TTestEnv* env, TTestSync* testSync) : Env(env) , TestSync(testSync) - { - } + { + } TContinueFunc Start() override { Sleep(TDuration::MilliSeconds(1)); @@ -77,8 +77,8 @@ Y_UNIT_TEST_SUITE(Spawn) { TSpawnTask(TTestEnv* env, TTestSync* testSync) : Env(env) , TestSync(testSync) - { - } + { + } TSubtaskCompletion SubtaskCompletion; @@ -109,12 +109,12 @@ Y_UNIT_TEST_SUITE(Spawn) { TTestSync* const TestSync; unsigned I; - TSpawnLongTask(TTestEnv* env, TTestSync* testSync) - : Env(env) - , TestSync(testSync) - , I(0) - { - } + TSpawnLongTask(TTestEnv* env, TTestSync* testSync) + : Env(env) + , TestSync(testSync) + , I(0) + { + } std::array<TSubtaskCompletion, 3> Subtasks; diff --git a/library/cpp/messagebus/rain_check/core/task.cpp b/library/cpp/messagebus/rain_check/core/task.cpp index a098437d53..3c5c609d07 100644 --- a/library/cpp/messagebus/rain_check/core/task.cpp +++ b/library/cpp/messagebus/rain_check/core/task.cpp @@ -11,12 +11,12 @@ using namespace NRainCheck::NPrivate; using namespace NActor; namespace { - Y_POD_STATIC_THREAD(TTaskRunnerBase*) - ThreadCurrentTask; + Y_POD_STATIC_THREAD(TTaskRunnerBase*) + ThreadCurrentTask; } -void TNopSubtaskListener::SetDone() { -} +void TNopSubtaskListener::SetDone() { +} TNopSubtaskListener TNopSubtaskListener::Instance; @@ -46,12 +46,12 @@ namespace { ~TRunningInThisThreadGuard() { Y_ASSERT(ThreadCurrentTask == Task); - ThreadCurrentTask = nullptr; + ThreadCurrentTask = nullptr; } }; } -void NRainCheck::TTaskRunnerBase::Act(NActor::TDefaultTag) { +void NRainCheck::TTaskRunnerBase::Act(NActor::TDefaultTag) { Y_ASSERT(RefCount() > 0); TRunningInThisThreadGuard g(this); @@ -90,16 +90,16 @@ void NRainCheck::TTaskRunnerBase::Act(NActor::TDefaultTag) { } } -bool TTaskRunnerBase::IsRunningInThisThread() const { +bool TTaskRunnerBase::IsRunningInThisThread() const { return ThreadCurrentTask == this; } -TSubtaskCompletion::~TSubtaskCompletion() { +TSubtaskCompletion::~TSubtaskCompletion() { ESubtaskState state = State.Get(); Y_ASSERT(state == CREATED || state == DONE || state == CANCELED); } -void TSubtaskCompletion::FireCompletionCallback(ITaskBase* task) { +void TSubtaskCompletion::FireCompletionCallback(ITaskBase* task) { Y_ASSERT(IsComplete()); if (!!CompletionFunc) { @@ -111,7 +111,7 @@ void TSubtaskCompletion::FireCompletionCallback(ITaskBase* task) { } } -void NRainCheck::TSubtaskCompletion::Cancel() { +void NRainCheck::TSubtaskCompletion::Cancel() { for (;;) { ESubtaskState state = State.Get(); if (state == CREATED && State.CompareAndSet(CREATED, CANCELED)) { @@ -129,7 +129,7 @@ void NRainCheck::TSubtaskCompletion::Cancel() { } } -void TSubtaskCompletion::SetRunning(TTaskRunnerBase* parent) { +void TSubtaskCompletion::SetRunning(TTaskRunnerBase* parent) { Y_ASSERT(!TaskRunner); Y_ASSERT(!!parent); @@ -150,7 +150,7 @@ void TSubtaskCompletion::SetRunning(TTaskRunnerBase* parent) { } } -void TSubtaskCompletion::SetDone() { +void TSubtaskCompletion::SetDone() { Y_ASSERT(!!TaskRunner); TTaskRunnerBase* temp = TaskRunner; TaskRunner = nullptr; @@ -194,11 +194,11 @@ void NRainCheck::TTaskRunnerBase::ReleaseRef() } #endif -void TTaskRunnerBase::AssertInThisThread() const { +void TTaskRunnerBase::AssertInThisThread() const { Y_ASSERT(IsRunningInThisThread()); } -TTaskRunnerBase* TTaskRunnerBase::CurrentTask() { +TTaskRunnerBase* TTaskRunnerBase::CurrentTask() { Y_VERIFY(!!ThreadCurrentTask); return ThreadCurrentTask; } diff --git a/library/cpp/messagebus/rain_check/core/task.h b/library/cpp/messagebus/rain_check/core/task.h index 7d8778bcda..16a0b3cb19 100644 --- a/library/cpp/messagebus/rain_check/core/task.h +++ b/library/cpp/messagebus/rain_check/core/task.h @@ -14,8 +14,8 @@ namespace NRainCheck { struct ISubtaskListener { virtual void SetDone() = 0; - virtual ~ISubtaskListener() { - } + virtual ~ISubtaskListener() { + } }; struct TNopSubtaskListener: public ISubtaskListener { @@ -33,17 +33,17 @@ namespace NRainCheck { public: TSubtaskCompletionFunc() : Func(nullptr) - { - } + { + } TSubtaskCompletionFunc(void*) : Func(nullptr) - { - } + { + } template <typename TTask> TSubtaskCompletionFunc(void (TTask::*func)(TSubtaskCompletion*)) - : Func((TFunc)func) + : Func((TFunc)func) { static_assert((std::is_base_of<ITaskBase, TTask>::value), "expect (std::is_base_of<ITaskBase, TTask>::value)"); } @@ -56,13 +56,13 @@ namespace NRainCheck { template <typename T> class TTaskFuture; -#define SUBTASK_STATE_MAP(XX) \ - XX(CREATED, "Initial") \ - XX(RUNNING, "Running") \ - XX(DONE, "Completed") \ - XX(CANCEL_REQUESTED, "Cancel requested, but still executing") \ - XX(CANCELED, "Canceled") \ - /**/ +#define SUBTASK_STATE_MAP(XX) \ + XX(CREATED, "Initial") \ + XX(RUNNING, "Running") \ + XX(DONE, "Completed") \ + XX(CANCEL_REQUESTED, "Cancel requested, but still executing") \ + XX(CANCELED, "Canceled") \ + /**/ enum ESubtaskState { SUBTASK_STATE_MAP(ENUM_VALUE_GEN_NO_VALUE) @@ -70,38 +70,38 @@ namespace NRainCheck { ENUM_TO_STRING(ESubtaskState, SUBTASK_STATE_MAP) - class TSubtaskCompletion : TNonCopyable, public ISubtaskListener { + class TSubtaskCompletion : TNonCopyable, public ISubtaskListener { friend struct TTaskAccessor; - + private: TAtomicBox<ESubtaskState> State; TTaskRunnerBase* volatile TaskRunner; TSubtaskCompletionFunc CompletionFunc; - + public: - TSubtaskCompletion() - : State(CREATED) - , TaskRunner() - { - } + TSubtaskCompletion() + : State(CREATED) + , TaskRunner() + { + } ~TSubtaskCompletion() override; // Either done or cancel requested or cancelled bool IsComplete() const { ESubtaskState state = State.Get(); switch (state) { - case RUNNING: - return false; - case DONE: - return true; - case CANCEL_REQUESTED: - return false; - case CANCELED: - return true; - case CREATED: - Y_FAIL("not started"); - default: - Y_FAIL("unknown value: %u", (unsigned)state); + case RUNNING: + return false; + case DONE: + return true; + case CANCEL_REQUESTED: + return false; + case CANCELED: + return true; + case CREATED: + Y_FAIL("not started"); + default: + Y_FAIL("unknown value: %u", (unsigned)state); } } @@ -128,7 +128,7 @@ namespace NRainCheck { }; // See ISimpleTask, ICoroTask - class TTaskRunnerBase: public TAtomicRefCount<TTaskRunnerBase>, public NActor::TActor<TTaskRunnerBase> { + class TTaskRunnerBase: public TAtomicRefCount<TTaskRunnerBase>, public NActor::TActor<TTaskRunnerBase> { friend class NActor::TActor<TTaskRunnerBase>; friend class TContinueFunc; friend struct TTaskAccessor; @@ -163,9 +163,9 @@ namespace NRainCheck { protected: //void RetainRef(); //void ReleaseRef(); - ITaskBase* GetImplBase() { - return Impl.Get(); - } + ITaskBase* GetImplBase() { + return Impl.Get(); + } private: // true if need to call again @@ -174,11 +174,11 @@ namespace NRainCheck { class ITaskBase { public: - virtual ~ITaskBase() { - } + virtual ~ITaskBase() { + } }; // Check that current method executed inside some task. bool AreWeInsideTask(); -} +} diff --git a/library/cpp/messagebus/rain_check/core/track.cpp b/library/cpp/messagebus/rain_check/core/track.cpp index 092a51a214..d704ee022a 100644 --- a/library/cpp/messagebus/rain_check/core/track.cpp +++ b/library/cpp/messagebus/rain_check/core/track.cpp @@ -3,7 +3,7 @@ using namespace NRainCheck; using namespace NRainCheck::NPrivate; -void TTaskTrackerReceipt::SetDone() { +void TTaskTrackerReceipt::SetDone() { TaskTracker->GetQueue<TTaskTrackerReceipt*>()->EnqueueAndSchedule(this); } @@ -11,12 +11,12 @@ TString TTaskTrackerReceipt::GetStatusSingleLine() { return Task->GetStatusSingleLine(); } -TTaskTracker::TTaskTracker(NActor::TExecutor* executor) - : NActor::TActor<TTaskTracker>(executor) +TTaskTracker::TTaskTracker(NActor::TExecutor* executor) + : NActor::TActor<TTaskTracker>(executor) { } -TTaskTracker::~TTaskTracker() { +TTaskTracker::~TTaskTracker() { Y_ASSERT(Tasks.Empty()); } @@ -47,7 +47,7 @@ void TTaskTracker::ProcessItem(NActor::TDefaultTag, NActor::TDefaultTag, TAsyncR status->SetResult(s); } -void TTaskTracker::Act(NActor::TDefaultTag) { +void TTaskTracker::Act(NActor::TDefaultTag) { GetQueue<TAsyncResult<TTaskTrackerStatus>*>()->DequeueAll(); GetQueue<ITaskFactory*>()->DequeueAll(); GetQueue<TTaskTrackerReceipt*>()->DequeueAll(); diff --git a/library/cpp/messagebus/rain_check/core/track.h b/library/cpp/messagebus/rain_check/core/track.h index d387de7574..88a4e6fd9e 100644 --- a/library/cpp/messagebus/rain_check/core/track.h +++ b/library/cpp/messagebus/rain_check/core/track.h @@ -16,18 +16,18 @@ namespace NRainCheck { namespace NPrivate { struct ITaskFactory { virtual TIntrusivePtr<TTaskRunnerBase> NewTask(ISubtaskListener*) = 0; - virtual ~ITaskFactory() { - } + virtual ~ITaskFactory() { + } }; struct TTaskTrackerReceipt: public ISubtaskListener, public TIntrusiveListItem<TTaskTrackerReceipt> { TTaskTracker* const TaskTracker; TIntrusivePtr<TTaskRunnerBase> Task; - TTaskTrackerReceipt(TTaskTracker* taskTracker) - : TaskTracker(taskTracker) - { - } + TTaskTrackerReceipt(TTaskTracker* taskTracker) + : TaskTracker(taskTracker) + { + } void SetDone() override; @@ -41,11 +41,11 @@ namespace NRainCheck { } class TTaskTracker - : public TAtomicRefCount<TTaskTracker>, - public NActor::TActor<TTaskTracker>, - public NActor::TQueueInActor<TTaskTracker, NPrivate::ITaskFactory*>, - public NActor::TQueueInActor<TTaskTracker, NPrivate::TTaskTrackerReceipt*>, - public NActor::TQueueInActor<TTaskTracker, TAsyncResult<NPrivate::TTaskTrackerStatus>*> { + : public TAtomicRefCount<TTaskTracker>, + public NActor::TActor<TTaskTracker>, + public NActor::TQueueInActor<TTaskTracker, NPrivate::ITaskFactory*>, + public NActor::TQueueInActor<TTaskTracker, NPrivate::TTaskTrackerReceipt*>, + public NActor::TQueueInActor<TTaskTracker, TAsyncResult<NPrivate::TTaskTrackerStatus>*> { friend struct NPrivate::TTaskTrackerReceipt; private: @@ -77,11 +77,11 @@ namespace NRainCheck { TEnv* const Env; TParam Param; - TTaskFactory(TEnv* env, TParam param) - : Env(env) - , Param(param) - { - } + TTaskFactory(TEnv* env, TParam param) + : Env(env) + , Param(param) + { + } TIntrusivePtr<TTaskRunnerBase> NewTask(ISubtaskListener* subtaskListener) override { return NRainCheck::SpawnTask<TTask>(Env, Param, subtaskListener).Get(); diff --git a/library/cpp/messagebus/rain_check/core/track_ut.cpp b/library/cpp/messagebus/rain_check/core/track_ut.cpp index 05f7de1319..ef76a7a2af 100644 --- a/library/cpp/messagebus/rain_check/core/track_ut.cpp +++ b/library/cpp/messagebus/rain_check/core/track_ut.cpp @@ -11,10 +11,10 @@ Y_UNIT_TEST_SUITE(TaskTracker) { struct TTaskForTracker: public ISimpleTask { TTestSync* const TestSync; - TTaskForTracker(TTestEnv*, TTestSync* testSync) - : TestSync(testSync) - { - } + TTaskForTracker(TTestEnv*, TTestSync* testSync) + : TestSync(testSync) + { + } TContinueFunc Start() override { TestSync->WaitForAndIncrement(0); diff --git a/library/cpp/messagebus/rain_check/http/client.cpp b/library/cpp/messagebus/rain_check/http/client.cpp index 5ef5ceeece..0fb8a93420 100644 --- a/library/cpp/messagebus/rain_check/http/client.cpp +++ b/library/cpp/messagebus/rain_check/http/client.cpp @@ -14,141 +14,141 @@ #include <util/stream/str.h> namespace NRainCheck { - class THttpCallback: public NNeh::IOnRecv { - public: - THttpCallback(NRainCheck::THttpFuture* future) - : Future(future) - { - Y_VERIFY(!!future, "future is NULL"); - } - - void OnRecv(NNeh::THandle& handle) override { - THolder<THttpCallback> self(this); - NNeh::TResponseRef response = handle.Get(); - Future->SetDoneAndSchedule(response); - } - - private: - NRainCheck::THttpFuture* const Future; - }; - - THttpFuture::THttpFuture() - : Task(nullptr) - , ErrorCode(THttpFuture::NoError) + class THttpCallback: public NNeh::IOnRecv { + public: + THttpCallback(NRainCheck::THttpFuture* future) + : Future(future) + { + Y_VERIFY(!!future, "future is NULL"); + } + + void OnRecv(NNeh::THandle& handle) override { + THolder<THttpCallback> self(this); + NNeh::TResponseRef response = handle.Get(); + Future->SetDoneAndSchedule(response); + } + + private: + NRainCheck::THttpFuture* const Future; + }; + + THttpFuture::THttpFuture() + : Task(nullptr) + , ErrorCode(THttpFuture::NoError) { } - THttpFuture::~THttpFuture() { + THttpFuture::~THttpFuture() { } - bool THttpFuture::HasError() const { - return (ErrorCode != THttpFuture::NoError); - } + bool THttpFuture::HasError() const { + return (ErrorCode != THttpFuture::NoError); + } - THttpFuture::EError THttpFuture::GetErrorCode() const { - return ErrorCode; - } + THttpFuture::EError THttpFuture::GetErrorCode() const { + return ErrorCode; + } - TString THttpFuture::GetErrorDescription() const { - return ErrorDescription; - } + TString THttpFuture::GetErrorDescription() const { + return ErrorDescription; + } - THttpClientService::THttpClientService() - : GetProtocol(NNeh::ProtocolFactory()->Protocol("http")) - , FullProtocol(NNeh::ProtocolFactory()->Protocol("full")) - { - Y_VERIFY(!!GetProtocol, "GET protocol is NULL."); - Y_VERIFY(!!FullProtocol, "POST protocol is NULL."); - } + THttpClientService::THttpClientService() + : GetProtocol(NNeh::ProtocolFactory()->Protocol("http")) + , FullProtocol(NNeh::ProtocolFactory()->Protocol("full")) + { + Y_VERIFY(!!GetProtocol, "GET protocol is NULL."); + Y_VERIFY(!!FullProtocol, "POST protocol is NULL."); + } - THttpClientService::~THttpClientService() { - } + THttpClientService::~THttpClientService() { + } - void THttpClientService::SendPost(TString addr, const TString& data, const THttpHeaders& headers, THttpFuture* future) { - Y_VERIFY(!!future, "future is NULL."); + void THttpClientService::SendPost(TString addr, const TString& data, const THttpHeaders& headers, THttpFuture* future) { + Y_VERIFY(!!future, "future is NULL."); - TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); - future->SetRunning(current); - future->Task = current; + TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); + future->SetRunning(current); + future->Task = current; - THolder<THttpCallback> callback(new THttpCallback(future)); - NNeh::TServiceStatRef stat; - try { + THolder<THttpCallback> callback(new THttpCallback(future)); + NNeh::TServiceStatRef stat; + try { NNeh::TMessage msg(addr.replace(0, NNeh::TParsedLocation(addr).Scheme.size(), "post"), data); - TStringStream headersText; - headers.OutTo(&headersText); - NNeh::NHttp::MakeFullRequest(msg, headersText.Str(), TString()); - FullProtocol->ScheduleRequest(msg, callback.Get(), stat); + TStringStream headersText; + headers.OutTo(&headersText); + NNeh::NHttp::MakeFullRequest(msg, headersText.Str(), TString()); + FullProtocol->ScheduleRequest(msg, callback.Get(), stat); Y_UNUSED(callback.Release()); - } catch (const TNetworkResolutionError& err) { - future->SetFail(THttpFuture::CantResolveNameError, err.AsStrBuf()); - } catch (const yexception& err) { - future->SetFail(THttpFuture::OtherError, err.AsStrBuf()); - } - } - - void THttpClientService::Send(const TString& request, THttpFuture* future) { - Y_VERIFY(!!future, "future is NULL."); - - TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); - future->SetRunning(current); - future->Task = current; - - THolder<THttpCallback> callback(new THttpCallback(future)); - NNeh::TServiceStatRef stat; - try { - GetProtocol->ScheduleRequest(NNeh::TMessage::FromString(request), - callback.Get(), - stat); + } catch (const TNetworkResolutionError& err) { + future->SetFail(THttpFuture::CantResolveNameError, err.AsStrBuf()); + } catch (const yexception& err) { + future->SetFail(THttpFuture::OtherError, err.AsStrBuf()); + } + } + + void THttpClientService::Send(const TString& request, THttpFuture* future) { + Y_VERIFY(!!future, "future is NULL."); + + TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); + future->SetRunning(current); + future->Task = current; + + THolder<THttpCallback> callback(new THttpCallback(future)); + NNeh::TServiceStatRef stat; + try { + GetProtocol->ScheduleRequest(NNeh::TMessage::FromString(request), + callback.Get(), + stat); Y_UNUSED(callback.Release()); - } catch (const TNetworkResolutionError& err) { - future->SetFail(THttpFuture::CantResolveNameError, err.AsStrBuf()); - } catch (const yexception& err) { - future->SetFail(THttpFuture::OtherError, err.AsStrBuf()); - } - } - - bool THttpFuture::HasHttpCode() const { - return !!HttpCode; - } - - bool THttpFuture::HasResponseBody() const { - return !!Response; + } catch (const TNetworkResolutionError& err) { + future->SetFail(THttpFuture::CantResolveNameError, err.AsStrBuf()); + } catch (const yexception& err) { + future->SetFail(THttpFuture::OtherError, err.AsStrBuf()); + } + } + + bool THttpFuture::HasHttpCode() const { + return !!HttpCode; } - ui32 THttpFuture::GetHttpCode() const { - Y_ASSERT(IsDone()); - Y_ASSERT(HasHttpCode()); + bool THttpFuture::HasResponseBody() const { + return !!Response; + } - return static_cast<ui32>(*HttpCode); - } - - TString THttpFuture::GetResponseBody() const { - Y_ASSERT(IsDone()); - Y_ASSERT(HasResponseBody()); - - return Response->Data; - } + ui32 THttpFuture::GetHttpCode() const { + Y_ASSERT(IsDone()); + Y_ASSERT(HasHttpCode()); - void THttpFuture::SetDoneAndSchedule(TAutoPtr<NNeh::TResponse> response) { - if (!response->IsError()) { - ErrorCode = THttpFuture::NoError; - HttpCode = HttpCodes::HTTP_OK; - } else { - ErrorCode = THttpFuture::BadHttpCodeError; - ErrorDescription = response->GetErrorText(); - - HttpCode = TryGetHttpCodeFromErrorDescription(ErrorDescription); - } - Response.Reset(response); - SetDone(); + return static_cast<ui32>(*HttpCode); } - void THttpFuture::SetFail(THttpFuture::EError errorCode, const TStringBuf& errorDescription) { - ErrorCode = errorCode; - ErrorDescription = errorDescription; - Response.Destroy(); - SetDone(); + TString THttpFuture::GetResponseBody() const { + Y_ASSERT(IsDone()); + Y_ASSERT(HasResponseBody()); + + return Response->Data; + } + + void THttpFuture::SetDoneAndSchedule(TAutoPtr<NNeh::TResponse> response) { + if (!response->IsError()) { + ErrorCode = THttpFuture::NoError; + HttpCode = HttpCodes::HTTP_OK; + } else { + ErrorCode = THttpFuture::BadHttpCodeError; + ErrorDescription = response->GetErrorText(); + + HttpCode = TryGetHttpCodeFromErrorDescription(ErrorDescription); + } + Response.Reset(response); + SetDone(); + } + + void THttpFuture::SetFail(THttpFuture::EError errorCode, const TStringBuf& errorDescription) { + ErrorCode = errorCode; + ErrorDescription = errorDescription; + Response.Destroy(); + SetDone(); } } diff --git a/library/cpp/messagebus/rain_check/http/client.h b/library/cpp/messagebus/rain_check/http/client.h index d4199c4c98..b7f822ae10 100644 --- a/library/cpp/messagebus/rain_check/http/client.h +++ b/library/cpp/messagebus/rain_check/http/client.h @@ -13,66 +13,66 @@ class THttpHeaders; namespace NNeh { - class IProtocol; - struct TResponse; -} + class IProtocol; + struct TResponse; +} namespace NRainCheck { - class THttpCallback; - class THttpClientService; + class THttpCallback; + class THttpClientService; - class THttpFuture: public TSubtaskCompletion { - public: - enum EError { - NoError = 0, + class THttpFuture: public TSubtaskCompletion { + public: + enum EError { + NoError = 0, - CantResolveNameError = 1, - BadHttpCodeError = 2, + CantResolveNameError = 1, + BadHttpCodeError = 2, - OtherError = 100 - }; + OtherError = 100 + }; - private: - friend class THttpCallback; - friend class THttpClientService; + private: + friend class THttpCallback; + friend class THttpClientService; - public: - THttpFuture(); - ~THttpFuture() override; + public: + THttpFuture(); + ~THttpFuture() override; - bool HasHttpCode() const; - bool HasResponseBody() const; + bool HasHttpCode() const; + bool HasResponseBody() const; - ui32 GetHttpCode() const; - TString GetResponseBody() const; + ui32 GetHttpCode() const; + TString GetResponseBody() const; - bool HasError() const; - EError GetErrorCode() const; - TString GetErrorDescription() const; + bool HasError() const; + EError GetErrorCode() const; + TString GetErrorDescription() const; - private: - void SetDoneAndSchedule(TAutoPtr<NNeh::TResponse> response); - void SetFail(EError errorCode, const TStringBuf& errorDescription); + private: + void SetDoneAndSchedule(TAutoPtr<NNeh::TResponse> response); + void SetFail(EError errorCode, const TStringBuf& errorDescription); - private: - TTaskRunnerBase* Task; - TMaybe<HttpCodes> HttpCode; - THolder<NNeh::TResponse> Response; - EError ErrorCode; - TString ErrorDescription; - }; + private: + TTaskRunnerBase* Task; + TMaybe<HttpCodes> HttpCode; + THolder<NNeh::TResponse> Response; + EError ErrorCode; + TString ErrorDescription; + }; - class THttpClientService { - public: - THttpClientService(); - virtual ~THttpClientService(); + class THttpClientService { + public: + THttpClientService(); + virtual ~THttpClientService(); - void Send(const TString& request, THttpFuture* future); - void SendPost(TString addr, const TString& data, const THttpHeaders& headers, THttpFuture* future); + void Send(const TString& request, THttpFuture* future); + void SendPost(TString addr, const TString& data, const THttpHeaders& headers, THttpFuture* future); - private: - NNeh::IProtocol* const GetProtocol; - NNeh::IProtocol* const FullProtocol; - }; + private: + NNeh::IProtocol* const GetProtocol; + NNeh::IProtocol* const FullProtocol; + }; -} +} diff --git a/library/cpp/messagebus/rain_check/http/client_ut.cpp b/library/cpp/messagebus/rain_check/http/client_ut.cpp index 1628114391..51d0296a37 100644 --- a/library/cpp/messagebus/rain_check/http/client_ut.cpp +++ b/library/cpp/messagebus/rain_check/http/client_ut.cpp @@ -28,116 +28,116 @@ using namespace NRainCheck; using namespace NBus::NTest; namespace { - class THttpClientEnv: public TTestEnvTemplate<THttpClientEnv> { - public: - THttpClientService HttpClientService; - }; - - const TString TEST_SERVICE = "test-service"; - const TString TEST_GET_PARAMS = "p=GET"; - const TString TEST_POST_PARAMS = "p=POST"; - const TString TEST_POST_HEADERS = "Content-Type: application/json\r\n"; - const TString TEST_GET_RECV = "GET was ok."; - const TString TEST_POST_RECV = "POST was ok."; - - TString BuildServiceLocation(ui32 port) { + class THttpClientEnv: public TTestEnvTemplate<THttpClientEnv> { + public: + THttpClientService HttpClientService; + }; + + const TString TEST_SERVICE = "test-service"; + const TString TEST_GET_PARAMS = "p=GET"; + const TString TEST_POST_PARAMS = "p=POST"; + const TString TEST_POST_HEADERS = "Content-Type: application/json\r\n"; + const TString TEST_GET_RECV = "GET was ok."; + const TString TEST_POST_RECV = "POST was ok."; + + TString BuildServiceLocation(ui32 port) { return Sprintf("http://*:%" PRIu32 "/%s", port, TEST_SERVICE.data()); - } + } - TString BuildPostServiceLocation(ui32 port) { + TString BuildPostServiceLocation(ui32 port) { return Sprintf("post://*:%" PRIu32 "/%s", port + 1, TEST_SERVICE.data()); - } - - TString BuildGetTestRequest(ui32 port) { - return BuildServiceLocation(port) + "?" + TEST_GET_PARAMS; - } - - class TSimpleServer { - public: - inline void ServeRequest(const NNeh::IRequestRef& req) { - NNeh::TData response; - if (req->Data() == TEST_GET_PARAMS) { - response.assign(TEST_GET_RECV.begin(), TEST_GET_RECV.end()); - } else { - response.assign(TEST_POST_RECV.begin(), TEST_POST_RECV.end()); - } - req->SendReply(response); - } - }; - - NNeh::IServicesRef RunServer(ui32 port, TSimpleServer& server) { - NNeh::IServicesRef runner = NNeh::CreateLoop(); - runner->Add(BuildServiceLocation(port), server); - runner->Add(BuildPostServiceLocation(port), server); - - try { - const int THR_POOL_SIZE = 2; - runner->ForkLoop(THR_POOL_SIZE); - } catch (...) { + } + + TString BuildGetTestRequest(ui32 port) { + return BuildServiceLocation(port) + "?" + TEST_GET_PARAMS; + } + + class TSimpleServer { + public: + inline void ServeRequest(const NNeh::IRequestRef& req) { + NNeh::TData response; + if (req->Data() == TEST_GET_PARAMS) { + response.assign(TEST_GET_RECV.begin(), TEST_GET_RECV.end()); + } else { + response.assign(TEST_POST_RECV.begin(), TEST_POST_RECV.end()); + } + req->SendReply(response); + } + }; + + NNeh::IServicesRef RunServer(ui32 port, TSimpleServer& server) { + NNeh::IServicesRef runner = NNeh::CreateLoop(); + runner->Add(BuildServiceLocation(port), server); + runner->Add(BuildPostServiceLocation(port), server); + + try { + const int THR_POOL_SIZE = 2; + runner->ForkLoop(THR_POOL_SIZE); + } catch (...) { Y_FAIL("Can't run server: %s", CurrentExceptionMessage().data()); } - return runner; + return runner; } - enum ERequestType { - RT_HTTP_GET = 0, - RT_HTTP_POST = 1 - }; - - using TTaskParam = std::pair<TIpPort, ERequestType>; - - class THttpClientTask: public ISimpleTask { - public: - THttpClientTask(THttpClientEnv* env, TTaskParam param) - : Env(env) - , ServerPort(param.first) - , ReqType(param.second) - { - } - - TContinueFunc Start() override { - switch (ReqType) { - case RT_HTTP_GET: { - TString getRequest = BuildGetTestRequest(ServerPort); - for (size_t i = 0; i < 3; ++i) { - Requests.push_back(new THttpFuture()); - Env->HttpClientService.Send(getRequest, Requests[i].Get()); - } - break; + enum ERequestType { + RT_HTTP_GET = 0, + RT_HTTP_POST = 1 + }; + + using TTaskParam = std::pair<TIpPort, ERequestType>; + + class THttpClientTask: public ISimpleTask { + public: + THttpClientTask(THttpClientEnv* env, TTaskParam param) + : Env(env) + , ServerPort(param.first) + , ReqType(param.second) + { + } + + TContinueFunc Start() override { + switch (ReqType) { + case RT_HTTP_GET: { + TString getRequest = BuildGetTestRequest(ServerPort); + for (size_t i = 0; i < 3; ++i) { + Requests.push_back(new THttpFuture()); + Env->HttpClientService.Send(getRequest, Requests[i].Get()); + } + break; } - case RT_HTTP_POST: { - TString servicePath = BuildPostServiceLocation(ServerPort); - TStringInput headersText(TEST_POST_HEADERS); - THttpHeaders headers(&headersText); - for (size_t i = 0; i < 3; ++i) { - Requests.push_back(new THttpFuture()); - Env->HttpClientService.SendPost(servicePath, TEST_POST_PARAMS, headers, Requests[i].Get()); - } - break; + case RT_HTTP_POST: { + TString servicePath = BuildPostServiceLocation(ServerPort); + TStringInput headersText(TEST_POST_HEADERS); + THttpHeaders headers(&headersText); + for (size_t i = 0; i < 3; ++i) { + Requests.push_back(new THttpFuture()); + Env->HttpClientService.SendPost(servicePath, TEST_POST_PARAMS, headers, Requests[i].Get()); + } + break; } } - - return &THttpClientTask::GotReplies; + + return &THttpClientTask::GotReplies; } - TContinueFunc GotReplies() { - const TString& TEST_OK_RECV = (ReqType == RT_HTTP_GET) ? TEST_GET_RECV : TEST_POST_RECV; - for (size_t i = 0; i < Requests.size(); ++i) { - UNIT_ASSERT_EQUAL(Requests[i]->GetHttpCode(), 200); - UNIT_ASSERT_EQUAL(Requests[i]->GetResponseBody(), TEST_OK_RECV); - } - - Env->TestSync.CheckAndIncrement(0); + TContinueFunc GotReplies() { + const TString& TEST_OK_RECV = (ReqType == RT_HTTP_GET) ? TEST_GET_RECV : TEST_POST_RECV; + for (size_t i = 0; i < Requests.size(); ++i) { + UNIT_ASSERT_EQUAL(Requests[i]->GetHttpCode(), 200); + UNIT_ASSERT_EQUAL(Requests[i]->GetResponseBody(), TEST_OK_RECV); + } - return nullptr; + Env->TestSync.CheckAndIncrement(0); + + return nullptr; } - THttpClientEnv* const Env; - const TIpPort ServerPort; - const ERequestType ReqType; + THttpClientEnv* const Env; + const TIpPort ServerPort; + const ERequestType ReqType; - TVector<TSimpleSharedPtr<THttpFuture>> Requests; - }; + TVector<TSimpleSharedPtr<THttpFuture>> Requests; + }; } // anonymous namespace @@ -175,12 +175,12 @@ Y_UNIT_TEST_SUITE(RainCheckHttpClient) { } Y_UNIT_TEST(HttpCodeExtraction) { - // Find "request failed(" string, then copy len("HTTP/1.X NNN") chars and try to convert NNN to HTTP code. + // Find "request failed(" string, then copy len("HTTP/1.X NNN") chars and try to convert NNN to HTTP code. -#define CHECK_VALID_LINE(line, code) \ - UNIT_ASSERT_NO_EXCEPTION(TryGetHttpCodeFromErrorDescription(line)); \ - UNIT_ASSERT(!!TryGetHttpCodeFromErrorDescription(line)); \ - UNIT_ASSERT_EQUAL(*TryGetHttpCodeFromErrorDescription(line), code) +#define CHECK_VALID_LINE(line, code) \ + UNIT_ASSERT_NO_EXCEPTION(TryGetHttpCodeFromErrorDescription(line)); \ + UNIT_ASSERT(!!TryGetHttpCodeFromErrorDescription(line)); \ + UNIT_ASSERT_EQUAL(*TryGetHttpCodeFromErrorDescription(line), code) CHECK_VALID_LINE(TStringBuf("library/cpp/neh/http.cpp:<LINE>: request failed(HTTP/1.0 200 Some random message"), 200); CHECK_VALID_LINE(TStringBuf("library/cpp/neh/http.cpp:<LINE>: request failed(HTTP/1.0 404 Some random message"), 404); @@ -189,9 +189,9 @@ Y_UNIT_TEST_SUITE(RainCheckHttpClient) { CHECK_VALID_LINE(TStringBuf("request failed(HTTP/1.1 2004 Some random message"), 200); #undef CHECK_VALID_LINE -#define CHECK_INVALID_LINE(line) \ - UNIT_ASSERT_NO_EXCEPTION(TryGetHttpCodeFromErrorDescription(line)); \ - UNIT_ASSERT(!TryGetHttpCodeFromErrorDescription(line)) +#define CHECK_INVALID_LINE(line) \ + UNIT_ASSERT_NO_EXCEPTION(TryGetHttpCodeFromErrorDescription(line)); \ + UNIT_ASSERT(!TryGetHttpCodeFromErrorDescription(line)) CHECK_INVALID_LINE(TStringBuf("library/cpp/neh/http.cpp:<LINE>: request failed(HTTP/1.1 1 Some random message")); CHECK_INVALID_LINE(TStringBuf("request failed(HTTP/1.0 asdf Some random message")); diff --git a/library/cpp/messagebus/rain_check/http/http_code_extractor.cpp b/library/cpp/messagebus/rain_check/http/http_code_extractor.cpp index 51d75762f6..b0925c5a6b 100644 --- a/library/cpp/messagebus/rain_check/http/http_code_extractor.cpp +++ b/library/cpp/messagebus/rain_check/http/http_code_extractor.cpp @@ -8,32 +8,32 @@ #include <util/string/cast.h> namespace NRainCheck { - TMaybe<HttpCodes> TryGetHttpCodeFromErrorDescription(const TStringBuf& errorMessage) { + TMaybe<HttpCodes> TryGetHttpCodeFromErrorDescription(const TStringBuf& errorMessage) { // Try to get HttpCode from library/cpp/neh response. // If response has HttpCode and it is not 200 OK, library/cpp/neh will send a message // "library/cpp/neh/http.cpp:<LINE>: request failed(<FIRST-HTTP-RESPONSE-LINE>)" // (see library/cpp/neh/http.cpp:625). So, we will try to parse this message and - // find out HttpCode in it. It is bad temporary solution, but we have no choice. + // find out HttpCode in it. It is bad temporary solution, but we have no choice. const TStringBuf SUBSTR = "request failed("; - const size_t SUBSTR_LEN = SUBSTR.size(); + const size_t SUBSTR_LEN = SUBSTR.size(); const size_t FIRST_LINE_LEN = TStringBuf("HTTP/1.X NNN").size(); - TMaybe<HttpCodes> httpCode; + TMaybe<HttpCodes> httpCode; - const size_t substrPos = errorMessage.find(SUBSTR); - if (substrPos != TStringBuf::npos) { - const TStringBuf firstLineStart = errorMessage.SubStr(substrPos + SUBSTR_LEN, FIRST_LINE_LEN); - try { - httpCode = static_cast<HttpCodes>(ParseHttpRetCode(firstLineStart)); - if (*httpCode < HTTP_CONTINUE || *httpCode >= HTTP_CODE_MAX) { - httpCode = Nothing(); - } - } catch (const TFromStringException& ex) { - // Can't parse HttpCode: it is OK, because ErrorDescription can be random string. + const size_t substrPos = errorMessage.find(SUBSTR); + if (substrPos != TStringBuf::npos) { + const TStringBuf firstLineStart = errorMessage.SubStr(substrPos + SUBSTR_LEN, FIRST_LINE_LEN); + try { + httpCode = static_cast<HttpCodes>(ParseHttpRetCode(firstLineStart)); + if (*httpCode < HTTP_CONTINUE || *httpCode >= HTTP_CODE_MAX) { + httpCode = Nothing(); + } + } catch (const TFromStringException& ex) { + // Can't parse HttpCode: it is OK, because ErrorDescription can be random string. } } - - return httpCode; + + return httpCode; } } diff --git a/library/cpp/messagebus/rain_check/http/http_code_extractor.h b/library/cpp/messagebus/rain_check/http/http_code_extractor.h index 33b565fa1c..322fe24fa0 100644 --- a/library/cpp/messagebus/rain_check/http/http_code_extractor.h +++ b/library/cpp/messagebus/rain_check/http/http_code_extractor.h @@ -10,7 +10,7 @@ namespace NRainCheck { // If response has HttpCode and it is not 200 OK, library/cpp/neh will send a message // "library/cpp/neh/http.cpp:<LINE>: request failed(<FIRST-HTTP-RESPONSE-LINE>)" // (see library/cpp/neh/http.cpp:625). So, we will try to parse this message and - // find out HttpCode in it. It is bad temporary solution, but we have no choice. - TMaybe<HttpCodes> TryGetHttpCodeFromErrorDescription(const TStringBuf& errorMessage); + // find out HttpCode in it. It is bad temporary solution, but we have no choice. + TMaybe<HttpCodes> TryGetHttpCodeFromErrorDescription(const TStringBuf& errorMessage); -} +} diff --git a/library/cpp/messagebus/rain_check/messagebus/messagebus_client.cpp b/library/cpp/messagebus/rain_check/messagebus/messagebus_client.cpp index daac8d9a99..d5d20ab2d6 100644 --- a/library/cpp/messagebus/rain_check/messagebus/messagebus_client.cpp +++ b/library/cpp/messagebus/rain_check/messagebus/messagebus_client.cpp @@ -4,17 +4,17 @@ using namespace NRainCheck; using namespace NBus; TBusClientService::TBusClientService( - const NBus::TBusSessionConfig& config, - NBus::TBusProtocol* proto, - NBus::TBusMessageQueue* queue) { + const NBus::TBusSessionConfig& config, + NBus::TBusProtocol* proto, + NBus::TBusMessageQueue* queue) { Session = queue->CreateSource(proto, this, config); } -TBusClientService::~TBusClientService() { +TBusClientService::~TBusClientService() { Session->Shutdown(); } -void TBusClientService::SendCommon(NBus::TBusMessage* message, const NBus::TNetAddr&, TBusFuture* future) { +void TBusClientService::SendCommon(NBus::TBusMessage* message, const NBus::TNetAddr&, TBusFuture* future) { TTaskRunnerBase* current = TTaskRunnerBase::CurrentTask(); future->SetRunning(current); @@ -29,8 +29,8 @@ void TBusClientService::SendCommon(NBus::TBusMessage* message, const NBus::TNetA } void TBusClientService::ProcessResultCommon(NBus::TBusMessageAutoPtr message, - const NBus::TNetAddr&, TBusFuture* future, - NBus::EMessageStatus status) { + const NBus::TNetAddr&, TBusFuture* future, + NBus::EMessageStatus status) { Y_UNUSED(message.Release()); if (status == NBus::MESSAGE_OK) { @@ -41,8 +41,8 @@ void TBusClientService::ProcessResultCommon(NBus::TBusMessageAutoPtr message, } void TBusClientService::SendOneWay( - NBus::TBusMessageAutoPtr message, const NBus::TNetAddr& addr, - TBusFuture* future) { + NBus::TBusMessageAutoPtr message, const NBus::TNetAddr& addr, + TBusFuture* future) { SendCommon(message.Get(), addr, future); EMessageStatus ok = Session->SendMessageOneWay(message.Get(), &addr, false); @@ -54,8 +54,8 @@ NBus::TBusClientSessionPtr TBusClientService::GetSessionForMonitoring() const { } void TBusClientService::Send( - TBusMessageAutoPtr message, const TNetAddr& addr, - TBusFuture* future) { + TBusMessageAutoPtr message, const TNetAddr& addr, + TBusFuture* future) { SendCommon(message.Get(), addr, future); EMessageStatus ok = Session->SendMessage(message.Get(), &addr, false); @@ -63,35 +63,35 @@ void TBusClientService::Send( } void TBusClientService::OnReply( - TAutoPtr<TBusMessage> request, - TAutoPtr<TBusMessage> response) { - TBusFuture* future = (TBusFuture*)request->Data; + TAutoPtr<TBusMessage> request, + TAutoPtr<TBusMessage> response) { + TBusFuture* future = (TBusFuture*)request->Data; Y_ASSERT(future->Request.Get() == request.Get()); Y_UNUSED(request.Release()); future->SetDoneAndSchedule(MESSAGE_OK, response); } void NRainCheck::TBusClientService::OnMessageSentOneWay( - TAutoPtr<NBus::TBusMessage> request) { - TBusFuture* future = (TBusFuture*)request->Data; + TAutoPtr<NBus::TBusMessage> request) { + TBusFuture* future = (TBusFuture*)request->Data; Y_ASSERT(future->Request.Get() == request.Get()); Y_UNUSED(request.Release()); future->SetDoneAndSchedule(MESSAGE_OK, nullptr); } void TBusClientService::OnError( - TAutoPtr<TBusMessage> message, NBus::EMessageStatus status) { + TAutoPtr<TBusMessage> message, NBus::EMessageStatus status) { if (message->Data == nullptr) { return; } - TBusFuture* future = (TBusFuture*)message->Data; + TBusFuture* future = (TBusFuture*)message->Data; Y_ASSERT(future->Request.Get() == message.Get()); Y_UNUSED(message.Release()); future->SetDoneAndSchedule(status, nullptr); } -void TBusFuture::SetDoneAndSchedule(EMessageStatus status, TAutoPtr<TBusMessage> response) { +void TBusFuture::SetDoneAndSchedule(EMessageStatus status, TAutoPtr<TBusMessage> response) { Status = status; Response.Reset(response.Release()); SetDone(); diff --git a/library/cpp/messagebus/rain_check/messagebus/messagebus_client.h b/library/cpp/messagebus/rain_check/messagebus/messagebus_client.h index 0a291cdea6..b47ec13408 100644 --- a/library/cpp/messagebus/rain_check/messagebus/messagebus_client.h +++ b/library/cpp/messagebus/rain_check/messagebus/messagebus_client.h @@ -7,24 +7,24 @@ namespace NRainCheck { class TBusFuture: public TSubtaskCompletion { friend class TBusClientService; - + private: THolder<NBus::TBusMessage> Request; THolder<NBus::TBusMessage> Response; NBus::EMessageStatus Status; - + private: TTaskRunnerBase* Task; void SetDoneAndSchedule(NBus::EMessageStatus, TAutoPtr<NBus::TBusMessage>); - + public: // TODO: add MESSAGE_UNDEFINED - TBusFuture() - : Status(NBus::MESSAGE_DONT_ASK) - , Task(nullptr) - { - } + TBusFuture() + : Status(NBus::MESSAGE_DONT_ASK) + , Task(nullptr) + { + } NBus::TBusMessage* GetRequest() const { return Request.Get(); @@ -44,7 +44,7 @@ namespace NRainCheck { class TBusClientService: private NBus::IBusClientHandler { private: NBus::TBusClientSessionPtr Session; - + public: TBusClientService(const NBus::TBusSessionConfig&, NBus::TBusProtocol*, NBus::TBusMessageQueue*); ~TBusClientService() override; diff --git a/library/cpp/messagebus/rain_check/messagebus/messagebus_client_ut.cpp b/library/cpp/messagebus/rain_check/messagebus/messagebus_client_ut.cpp index 1b3618558b..e63ba86be7 100644 --- a/library/cpp/messagebus/rain_check/messagebus/messagebus_client_ut.cpp +++ b/library/cpp/messagebus/rain_check/messagebus/messagebus_client_ut.cpp @@ -28,8 +28,8 @@ struct TMessageBusClientEnv: public TTestEnvTemplate<TMessageBusClientEnv> { TMessageBusClientEnv() : Queue(CreateMessageQueue(GetExecutor())) , BusClientService(TBusSessionConfig(), &Proto, Queue.Get()) - { - } + { + } }; Y_UNIT_TEST_SUITE(RainCheckMessageBusClient) { @@ -44,7 +44,7 @@ Y_UNIT_TEST_SUITE(RainCheckMessageBusClient) { { } - TVector<TSimpleSharedPtr<TBusFuture>> Requests; + TVector<TSimpleSharedPtr<TBusFuture>> Requests; TContinueFunc Start() override { for (unsigned i = 0; i < 3; ++i) { @@ -108,7 +108,7 @@ Y_UNIT_TEST_SUITE(RainCheckMessageBusClient) { { } - TVector<TSimpleSharedPtr<TBusFuture>> Requests; + TVector<TSimpleSharedPtr<TBusFuture>> Requests; TContinueFunc Start() override { Env->TestSync.CheckAndIncrement(0); diff --git a/library/cpp/messagebus/rain_check/messagebus/messagebus_server.cpp b/library/cpp/messagebus/rain_check/messagebus/messagebus_server.cpp index 5d4b13d664..1868cfa06e 100644 --- a/library/cpp/messagebus/rain_check/messagebus/messagebus_server.cpp +++ b/library/cpp/messagebus/rain_check/messagebus/messagebus_server.cpp @@ -9,9 +9,9 @@ TBusTaskStarter::TBusTaskStarter(TAutoPtr<ITaskFactory> taskFactory) { } -void TBusTaskStarter::OnMessage(NBus::TOnMessageContext& onMessage) { +void TBusTaskStarter::OnMessage(NBus::TOnMessageContext& onMessage) { TaskFactory->NewTask(onMessage); } -TBusTaskStarter::~TBusTaskStarter() { +TBusTaskStarter::~TBusTaskStarter() { } diff --git a/library/cpp/messagebus/rain_check/messagebus/messagebus_server.h b/library/cpp/messagebus/rain_check/messagebus/messagebus_server.h index 1334f05fe4..10fc8b0dc7 100644 --- a/library/cpp/messagebus/rain_check/messagebus/messagebus_server.h +++ b/library/cpp/messagebus/rain_check/messagebus/messagebus_server.h @@ -12,14 +12,14 @@ namespace NRainCheck { private: struct ITaskFactory { virtual void NewTask(NBus::TOnMessageContext&) = 0; - virtual ~ITaskFactory() { - } + virtual ~ITaskFactory() { + } }; THolder<ITaskFactory> TaskFactory; void OnMessage(NBus::TOnMessageContext&) override; - + public: TBusTaskStarter(TAutoPtr<ITaskFactory>); ~TBusTaskStarter() override; @@ -30,10 +30,10 @@ namespace NRainCheck { struct TTaskFactory: public ITaskFactory { TEnv* const Env; - TTaskFactory(TEnv* env) - : Env(env) - { - } + TTaskFactory(TEnv* env) + : Env(env) + { + } void NewTask(NBus::TOnMessageContext& context) override { SpawnTask<TTask, TEnv, NBus::TOnMessageContext&>(Env, context); diff --git a/library/cpp/messagebus/rain_check/messagebus/messagebus_server_ut.cpp b/library/cpp/messagebus/rain_check/messagebus/messagebus_server_ut.cpp index 7c11399f1b..9b3bd58ad7 100644 --- a/library/cpp/messagebus/rain_check/messagebus/messagebus_server_ut.cpp +++ b/library/cpp/messagebus/rain_check/messagebus/messagebus_server_ut.cpp @@ -19,7 +19,7 @@ Y_UNIT_TEST_SUITE(RainCheckMessageBusServer) { private: TMessageBusServerEnv* const Env; TOnMessageContext MessageContext; - + public: TSimpleServerTask(TMessageBusServerEnv* env, TOnMessageContext& messageContext) : Env(env) diff --git a/library/cpp/messagebus/rain_check/test/helper/misc.cpp b/library/cpp/messagebus/rain_check/test/helper/misc.cpp index c0fcb27252..b176878cab 100644 --- a/library/cpp/messagebus/rain_check/test/helper/misc.cpp +++ b/library/cpp/messagebus/rain_check/test/helper/misc.cpp @@ -4,7 +4,7 @@ using namespace NRainCheck; -void TSpawnNopTasksCoroTask::Run() { +void TSpawnNopTasksCoroTask::Run() { Y_VERIFY(Count <= Completion.size()); for (unsigned i = 0; i < Count; ++i) { SpawnSubtask<TNopCoroTask>(Env, &Completion[i], ""); @@ -13,7 +13,7 @@ void TSpawnNopTasksCoroTask::Run() { WaitForSubtasks(); } -TContinueFunc TSpawnNopTasksSimpleTask::Start() { +TContinueFunc TSpawnNopTasksSimpleTask::Start() { Y_VERIFY(Count <= Completion.size()); for (unsigned i = 0; i < Count; ++i) { SpawnSubtask<TNopSimpleTask>(Env, &Completion[i], ""); @@ -22,6 +22,6 @@ TContinueFunc TSpawnNopTasksSimpleTask::Start() { return &TSpawnNopTasksSimpleTask::Join; } -TContinueFunc TSpawnNopTasksSimpleTask::Join() { +TContinueFunc TSpawnNopTasksSimpleTask::Join() { return nullptr; } diff --git a/library/cpp/messagebus/rain_check/test/helper/misc.h b/library/cpp/messagebus/rain_check/test/helper/misc.h index 9150be4d2f..5d8648462d 100644 --- a/library/cpp/messagebus/rain_check/test/helper/misc.h +++ b/library/cpp/messagebus/rain_check/test/helper/misc.h @@ -6,8 +6,8 @@ namespace NRainCheck { struct TNopSimpleTask: public ISimpleTask { - TNopSimpleTask(IEnv*, const void*) { - } + TNopSimpleTask(IEnv*, const void*) { + } TContinueFunc Start() override { return nullptr; @@ -15,11 +15,11 @@ namespace NRainCheck { }; struct TNopCoroTask: public ICoroTask { - TNopCoroTask(IEnv*, const void*) { - } + TNopCoroTask(IEnv*, const void*) { + } - void Run() override { - } + void Run() override { + } }; struct TSpawnNopTasksCoroTask: public ICoroTask { @@ -29,8 +29,8 @@ namespace NRainCheck { TSpawnNopTasksCoroTask(IEnv* env, unsigned count) : Env(env) , Count(count) - { - } + { + } std::array<TSubtaskCompletion, 2> Completion; @@ -44,8 +44,8 @@ namespace NRainCheck { TSpawnNopTasksSimpleTask(IEnv* env, unsigned count) : Env(env) , Count(count) - { - } + { + } std::array<TSubtaskCompletion, 2> Completion; diff --git a/library/cpp/messagebus/rain_check/test/perftest/perftest.cpp b/library/cpp/messagebus/rain_check/test/perftest/perftest.cpp index 22edbd8c6b..c71499e84a 100644 --- a/library/cpp/messagebus/rain_check/test/perftest/perftest.cpp +++ b/library/cpp/messagebus/rain_check/test/perftest/perftest.cpp @@ -16,17 +16,17 @@ struct TRainCheckPerftestEnv: public TSimpleEnvTemplate<TRainCheckPerftestEnv> { TRainCheckPerftestEnv() : TSimpleEnvTemplate<TRainCheckPerftestEnv>(4) , SubtasksPerTask(1000) - { - } + { + } }; struct TCoroOuter: public ICoroTask { TRainCheckPerftestEnv* const Env; - TCoroOuter(TRainCheckPerftestEnv* env) - : Env(env) - { - } + TCoroOuter(TRainCheckPerftestEnv* env) + : Env(env) + { + } void Run() override { for (;;) { @@ -63,10 +63,10 @@ struct TCoroOuter: public ICoroTask { struct TSimpleOuter: public ISimpleTask { TRainCheckPerftestEnv* const Env; - TSimpleOuter(TRainCheckPerftestEnv* env, const void*) - : Env(env) - { - } + TSimpleOuter(TRainCheckPerftestEnv* env, const void*) + : Env(env) + { + } TInstant StartInstant; unsigned Count; @@ -119,10 +119,10 @@ struct TSimpleOuter: public ISimpleTask { struct TReproduceCrashTask: public ISimpleTask { TRainCheckPerftestEnv* const Env; - TReproduceCrashTask(TRainCheckPerftestEnv* env) - : Env(env) - { - } + TReproduceCrashTask(TRainCheckPerftestEnv* env) + : Env(env) + { + } std::array<TSubtaskCompletion, SUBTASKS> Completion; diff --git a/library/cpp/messagebus/ref_counted.h b/library/cpp/messagebus/ref_counted.h index 29b87764e3..c1c4b95c52 100644 --- a/library/cpp/messagebus/ref_counted.h +++ b/library/cpp/messagebus/ref_counted.h @@ -1,6 +1,6 @@ #pragma once -class TAtomicRefCountedObject: public TAtomicRefCount<TAtomicRefCountedObject> { - virtual ~TAtomicRefCountedObject() { - } +class TAtomicRefCountedObject: public TAtomicRefCount<TAtomicRefCountedObject> { + virtual ~TAtomicRefCountedObject() { + } }; diff --git a/library/cpp/messagebus/remote_client_connection.cpp b/library/cpp/messagebus/remote_client_connection.cpp index 8c7a6db3a8..5006869683 100644 --- a/library/cpp/messagebus/remote_client_connection.cpp +++ b/library/cpp/messagebus/remote_client_connection.cpp @@ -33,7 +33,7 @@ TBusMessage* TRemoteClientConnection::PopAck(TBusKey id) { return AckMessages.Pop(id); } -SOCKET TRemoteClientConnection::CreateSocket(const TNetAddr& addr) { +SOCKET TRemoteClientConnection::CreateSocket(const TNetAddr& addr) { SOCKET handle = socket(addr.Addr()->sa_family, SOCK_STREAM, 0); Y_VERIFY(handle != INVALID_SOCKET, "failed to create socket: %s", LastSystemErrorText()); @@ -138,33 +138,33 @@ void TRemoteClientConnection::BeforeTryWrite() { TimeoutMessages(); } -namespace NBus { - namespace NPrivate { - class TInvokeOnReply: public IWorkItem { - private: - TRemoteClientSession* RemoteClientSession; - TNonDestroyingHolder<TBusMessage> Request; - TBusMessagePtrAndHeader Response; - - public: - TInvokeOnReply(TRemoteClientSession* session, - TNonDestroyingAutoPtr<TBusMessage> request, TBusMessagePtrAndHeader& response) - : RemoteClientSession(session) - , Request(request) - { - Response.Swap(response); - } - - void DoWork() override { - THolder<TInvokeOnReply> holder(this); - RemoteClientSession->ReleaseInFlightAndCallOnReply(Request.Release(), Response); - // TODO: TRemoteClientSessionSemaphore should be enough - RemoteClientSession->JobCount.Decrement(); - } - }; - - } -} +namespace NBus { + namespace NPrivate { + class TInvokeOnReply: public IWorkItem { + private: + TRemoteClientSession* RemoteClientSession; + TNonDestroyingHolder<TBusMessage> Request; + TBusMessagePtrAndHeader Response; + + public: + TInvokeOnReply(TRemoteClientSession* session, + TNonDestroyingAutoPtr<TBusMessage> request, TBusMessagePtrAndHeader& response) + : RemoteClientSession(session) + , Request(request) + { + Response.Swap(response); + } + + void DoWork() override { + THolder<TInvokeOnReply> holder(this); + RemoteClientSession->ReleaseInFlightAndCallOnReply(Request.Release(), Response); + // TODO: TRemoteClientSessionSemaphore should be enough + RemoteClientSession->JobCount.Decrement(); + } + }; + + } +} void TRemoteClientConnection::ProcessReplyQueue() { if (AtomicGet(WriterData.Down)) { diff --git a/library/cpp/messagebus/remote_client_connection.h b/library/cpp/messagebus/remote_client_connection.h index fe80b7d2f9..95061f1f36 100644 --- a/library/cpp/messagebus/remote_client_connection.h +++ b/library/cpp/messagebus/remote_client_connection.h @@ -7,59 +7,59 @@ #include <util/generic/object_counter.h> -namespace NBus { - namespace NPrivate { - class TRemoteClientConnection: public TRemoteConnection, public TBusClientConnection { - friend class TRemoteConnection; - friend struct TBusSessionImpl; - friend class TRemoteClientSession; +namespace NBus { + namespace NPrivate { + class TRemoteClientConnection: public TRemoteConnection, public TBusClientConnection { + friend class TRemoteConnection; + friend struct TBusSessionImpl; + friend class TRemoteClientSession; - private: - TObjectCounter<TRemoteClientConnection> ObjectCounter; + private: + TObjectCounter<TRemoteClientConnection> ObjectCounter; - TSyncAckMessages AckMessages; + TSyncAckMessages AckMessages; - TLocalTasks TimeToTimeoutMessages; + TLocalTasks TimeToTimeoutMessages; - IBusClientHandler* const ClientHandler; + IBusClientHandler* const ClientHandler; - public: - TRemoteClientConnection(TRemoteClientSessionPtr session, ui64 id, TNetAddr addr); + public: + TRemoteClientConnection(TRemoteClientSessionPtr session, ui64 id, TNetAddr addr); - inline TRemoteClientSession* GetSession(); + inline TRemoteClientSession* GetSession(); - SOCKET CreateSocket(const TNetAddr& addr); + SOCKET CreateSocket(const TNetAddr& addr); - void TryConnect() override; + void TryConnect() override; - void HandleEvent(SOCKET socket, void* cookie) override; + void HandleEvent(SOCKET socket, void* cookie) override; - TBusMessage* PopAck(TBusKey id); + TBusMessage* PopAck(TBusKey id); - void WriterFillStatus() override; + void WriterFillStatus() override; - void ClearOutgoingQueue(TMessagesPtrs& result, bool reconnect) override; + void ClearOutgoingQueue(TMessagesPtrs& result, bool reconnect) override; - void BeforeTryWrite() override; + void BeforeTryWrite() override; - void ProcessReplyQueue(); + void ProcessReplyQueue(); - void MessageSent(TArrayRef<TBusMessagePtrAndHeader> messages) override; + void MessageSent(TArrayRef<TBusMessagePtrAndHeader> messages) override; - void TimeoutMessages(); + void TimeoutMessages(); - void ScheduleTimeoutMessages(); + void ScheduleTimeoutMessages(); - void ReaderProcessMessageUnknownVersion(TArrayRef<const char> dataRef) override; + void ReaderProcessMessageUnknownVersion(TArrayRef<const char> dataRef) override; - EMessageStatus SendMessage(TBusMessage* pMes, bool wait) override; + EMessageStatus SendMessage(TBusMessage* pMes, bool wait) override; - EMessageStatus SendMessageOneWay(TBusMessage* pMes, bool wait) override; + EMessageStatus SendMessageOneWay(TBusMessage* pMes, bool wait) override; - EMessageStatus SendMessageImpl(TBusMessage*, bool wait, bool oneWay); + EMessageStatus SendMessageImpl(TBusMessage*, bool wait, bool oneWay); - void OpenConnection() override; - }; + void OpenConnection() override; + }; - } -} + } +} diff --git a/library/cpp/messagebus/remote_client_session.cpp b/library/cpp/messagebus/remote_client_session.cpp index 3bc421944f..77051f1099 100644 --- a/library/cpp/messagebus/remote_client_session.cpp +++ b/library/cpp/messagebus/remote_client_session.cpp @@ -14,8 +14,8 @@ using namespace NBus; using namespace NBus::NPrivate; TRemoteClientSession::TRemoteClientSession(TBusMessageQueue* queue, - TBusProtocol* proto, IBusClientHandler* handler, - const TBusClientSessionConfig& config, const TString& name) + TBusProtocol* proto, IBusClientHandler* handler, + const TBusClientSessionConfig& config, const TString& name) : TBusSessionImpl(true, queue, proto, handler, config, name) , ClientRemoteInFlight(config.MaxInFlight, "ClientRemoteInFlight") , ClientHandler(handler) @@ -27,7 +27,7 @@ TRemoteClientSession::~TRemoteClientSession() { } void TRemoteClientSession::OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& newMsg) { - TAutoPtr<TVectorSwaps<TBusMessagePtrAndHeader>> temp(new TVectorSwaps<TBusMessagePtrAndHeader>); + TAutoPtr<TVectorSwaps<TBusMessagePtrAndHeader>> temp(new TVectorSwaps<TBusMessagePtrAndHeader>); temp->swap(newMsg); c->ReplyQueue.EnqueueAll(temp); c->ScheduleWrite(); @@ -46,7 +46,7 @@ EMessageStatus TRemoteClientSession::SendMessageImpl(TBusMessage* msg, const TNe msg->ReplyTo = resolvedAddr; - TRemoteConnectionPtr c = ((TBusSessionImpl*)this)->GetConnection(resolvedAddr, true); + TRemoteConnectionPtr c = ((TBusSessionImpl*)this)->GetConnection(resolvedAddr, true); Y_ASSERT(!!c); return CheckedCast<TRemoteClientConnection*>(c.Get())->SendMessageImpl(msg, wait, oneWay); @@ -103,7 +103,7 @@ void TRemoteClientSession::ReleaseInFlightAndCallOnReply(TNonDestroyingAutoPtr<T } } -EMessageStatus TRemoteClientSession::GetMessageDestination(TBusMessage* mess, const TNetAddr* addrp, TBusSocketAddr* dest) { +EMessageStatus TRemoteClientSession::GetMessageDestination(TBusMessage* mess, const TNetAddr* addrp, TBusSocketAddr* dest) { if (addrp) { *dest = *addrp; } else { @@ -123,5 +123,5 @@ void TRemoteClientSession::OpenConnection(const TNetAddr& addr) { TBusClientConnectionPtr TRemoteClientSession::GetConnection(const TNetAddr& addr) { // TODO: GetConnection should not open - return CheckedCast<TRemoteClientConnection*>(((TBusSessionImpl*)this)->GetConnection(addr, true).Get()); + return CheckedCast<TRemoteClientConnection*>(((TBusSessionImpl*)this)->GetConnection(addr, true).Get()); } diff --git a/library/cpp/messagebus/remote_client_session.h b/library/cpp/messagebus/remote_client_session.h index 7160d0dae9..7378fcb0bd 100644 --- a/library/cpp/messagebus/remote_client_session.h +++ b/library/cpp/messagebus/remote_client_session.h @@ -8,52 +8,52 @@ #ifdef _MSC_VER #pragma warning(push) -#pragma warning(disable : 4250) // 'NBus::NPrivate::TRemoteClientSession' : inherits 'NBus::NPrivate::TBusSessionImpl::NBus::NPrivate::TBusSessionImpl::GetConfig' via dominance +#pragma warning(disable : 4250) // 'NBus::NPrivate::TRemoteClientSession' : inherits 'NBus::NPrivate::TBusSessionImpl::NBus::NPrivate::TBusSessionImpl::GetConfig' via dominance #endif -namespace NBus { - namespace NPrivate { - using TRemoteClientSessionPtr = TIntrusivePtr<TRemoteClientSession>; +namespace NBus { + namespace NPrivate { + using TRemoteClientSessionPtr = TIntrusivePtr<TRemoteClientSession>; - class TRemoteClientSession: public TBusClientSession, public TBusSessionImpl { - friend class TRemoteClientConnection; - friend class TInvokeOnReply; + class TRemoteClientSession: public TBusClientSession, public TBusSessionImpl { + friend class TRemoteClientConnection; + friend class TInvokeOnReply; - public: - TObjectCounter<TRemoteClientSession> ObjectCounter; + public: + TObjectCounter<TRemoteClientSession> ObjectCounter; - TRemoteClientSessionSemaphore ClientRemoteInFlight; - IBusClientHandler* const ClientHandler; + TRemoteClientSessionSemaphore ClientRemoteInFlight; + IBusClientHandler* const ClientHandler; - public: - TRemoteClientSession(TBusMessageQueue* queue, TBusProtocol* proto, - IBusClientHandler* handler, - const TBusSessionConfig& config, const TString& name); + public: + TRemoteClientSession(TBusMessageQueue* queue, TBusProtocol* proto, + IBusClientHandler* handler, + const TBusSessionConfig& config, const TString& name); - ~TRemoteClientSession() override; + ~TRemoteClientSession() override; - void OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& newMsg) override; + void OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& newMsg) override; - EMessageStatus SendMessageImpl(TBusMessage* msg, const TNetAddr* addr, bool wait, bool oneWay); - EMessageStatus SendMessage(TBusMessage* msg, const TNetAddr* addr = nullptr, bool wait = false) override; - EMessageStatus SendMessageOneWay(TBusMessage* msg, const TNetAddr* addr = nullptr, bool wait = false) override; + EMessageStatus SendMessageImpl(TBusMessage* msg, const TNetAddr* addr, bool wait, bool oneWay); + EMessageStatus SendMessage(TBusMessage* msg, const TNetAddr* addr = nullptr, bool wait = false) override; + EMessageStatus SendMessageOneWay(TBusMessage* msg, const TNetAddr* addr = nullptr, bool wait = false) override; - int GetInFlight() const noexcept override; - void FillStatus() override; - void AcquireInFlight(TArrayRef<TBusMessage* const> messages); - void ReleaseInFlight(TArrayRef<TBusMessage* const> messages); - void ReleaseInFlightAndCallOnReply(TNonDestroyingAutoPtr<TBusMessage> request, TBusMessagePtrAndHeader& response); + int GetInFlight() const noexcept override; + void FillStatus() override; + void AcquireInFlight(TArrayRef<TBusMessage* const> messages); + void ReleaseInFlight(TArrayRef<TBusMessage* const> messages); + void ReleaseInFlightAndCallOnReply(TNonDestroyingAutoPtr<TBusMessage> request, TBusMessagePtrAndHeader& response); - EMessageStatus GetMessageDestination(TBusMessage* mess, const TNetAddr* addrp, TBusSocketAddr* dest); + EMessageStatus GetMessageDestination(TBusMessage* mess, const TNetAddr* addrp, TBusSocketAddr* dest); - void OpenConnection(const TNetAddr&) override; + void OpenConnection(const TNetAddr&) override; - TBusClientConnectionPtr GetConnection(const TNetAddr&) override; - }; + TBusClientConnectionPtr GetConnection(const TNetAddr&) override; + }; #ifdef _MSC_VER #pragma warning(pop) #endif - } -} + } +} diff --git a/library/cpp/messagebus/remote_client_session_semaphore.cpp b/library/cpp/messagebus/remote_client_session_semaphore.cpp index f877ed4257..fce7349a36 100644 --- a/library/cpp/messagebus/remote_client_session_semaphore.cpp +++ b/library/cpp/messagebus/remote_client_session_semaphore.cpp @@ -16,7 +16,7 @@ TRemoteClientSessionSemaphore::TRemoteClientSessionSemaphore(TAtomicBase limit, Y_UNUSED(Name); } -TRemoteClientSessionSemaphore::~TRemoteClientSessionSemaphore() { +TRemoteClientSessionSemaphore::~TRemoteClientSessionSemaphore() { Y_VERIFY(AtomicGet(Current) == 0); } diff --git a/library/cpp/messagebus/remote_client_session_semaphore.h b/library/cpp/messagebus/remote_client_session_semaphore.h index 286ca3c86f..955440b25f 100644 --- a/library/cpp/messagebus/remote_client_session_semaphore.h +++ b/library/cpp/messagebus/remote_client_session_semaphore.h @@ -7,36 +7,36 @@ #include <util/system/condvar.h> #include <util/system/mutex.h> -namespace NBus { - namespace NPrivate { - class TRemoteClientSessionSemaphore: public TComplexConditionSemaphore<TRemoteClientSessionSemaphore> { - private: - const char* const Name; - - TAtomicBase const Limit; - TAtomic Current; - TAtomic StopSignal; - - public: - TRemoteClientSessionSemaphore(TAtomicBase limit, const char* name = "unnamed"); - ~TRemoteClientSessionSemaphore(); - - TAtomicBase GetCurrent() const { - return AtomicGet(Current); - } - - void Acquire(); - bool TryAcquire(); - void Increment(); - void IncrementMultiple(TAtomicBase count); - bool TryWait(); - void Release(); - void ReleaseMultiple(TAtomicBase count); - void Stop(); - - private: - void CheckNeedToUnlock(); - }; - - } -} +namespace NBus { + namespace NPrivate { + class TRemoteClientSessionSemaphore: public TComplexConditionSemaphore<TRemoteClientSessionSemaphore> { + private: + const char* const Name; + + TAtomicBase const Limit; + TAtomic Current; + TAtomic StopSignal; + + public: + TRemoteClientSessionSemaphore(TAtomicBase limit, const char* name = "unnamed"); + ~TRemoteClientSessionSemaphore(); + + TAtomicBase GetCurrent() const { + return AtomicGet(Current); + } + + void Acquire(); + bool TryAcquire(); + void Increment(); + void IncrementMultiple(TAtomicBase count); + bool TryWait(); + void Release(); + void ReleaseMultiple(TAtomicBase count); + void Stop(); + + private: + void CheckNeedToUnlock(); + }; + + } +} diff --git a/library/cpp/messagebus/remote_connection.cpp b/library/cpp/messagebus/remote_connection.cpp index 22932569db..ca4a66f68a 100644 --- a/library/cpp/messagebus/remote_connection.cpp +++ b/library/cpp/messagebus/remote_connection.cpp @@ -20,955 +20,955 @@ using namespace NActor; using namespace NBus; using namespace NBus::NPrivate; -namespace NBus { - namespace NPrivate { - TRemoteConnection::TRemoteConnection(TRemoteSessionPtr session, ui64 connectionId, TNetAddr addr) - : TActor<TRemoteConnection, TWriterTag>(session->Queue->WorkQueue.Get()) - , TActor<TRemoteConnection, TReaderTag>(session->Queue->WorkQueue.Get()) - , TScheduleActor<TRemoteConnection, TWriterTag>(&session->Queue->Scheduler) - , Session(session) - , Proto(session->Proto) - , Config(session->Config) - , RemovedFromSession(false) - , ConnectionId(connectionId) - , PeerAddr(addr) - , PeerAddrSocketAddr(addr) - , CreatedTime(TInstant::Now()) - , ReturnConnectFailedImmediately(false) - , GranStatus(Config.Secret.StatusFlushPeriod) - , QuotaMsg(!Session->IsSource_, Config.PerConnectionMaxInFlight, 0) - , QuotaBytes(!Session->IsSource_, Config.PerConnectionMaxInFlightBySize, 0) - , MaxBufferSize(session->Config.MaxBufferSize) - , ShutdownReason(MESSAGE_OK) - { - WriterData.Status.ConnectionId = connectionId; - WriterData.Status.PeerAddr = PeerAddr; - ReaderData.Status.ConnectionId = connectionId; - - const TInstant now = TInstant::Now(); - - WriterFillStatus(); - - GranStatus.Writer.Update(WriterData.Status, now, true); - GranStatus.Reader.Update(ReaderData.Status, now, true); - } - - TRemoteConnection::~TRemoteConnection() { - Y_VERIFY(ReplyQueue.IsEmpty()); - } - - TRemoteConnection::TWriterData::TWriterData() - : Down(0) - , SocketVersion(0) - , InFlight(0) - , AwakeFlags(0) - , State(WRITER_FILLING) - { - } - - TRemoteConnection::TWriterData::~TWriterData() { +namespace NBus { + namespace NPrivate { + TRemoteConnection::TRemoteConnection(TRemoteSessionPtr session, ui64 connectionId, TNetAddr addr) + : TActor<TRemoteConnection, TWriterTag>(session->Queue->WorkQueue.Get()) + , TActor<TRemoteConnection, TReaderTag>(session->Queue->WorkQueue.Get()) + , TScheduleActor<TRemoteConnection, TWriterTag>(&session->Queue->Scheduler) + , Session(session) + , Proto(session->Proto) + , Config(session->Config) + , RemovedFromSession(false) + , ConnectionId(connectionId) + , PeerAddr(addr) + , PeerAddrSocketAddr(addr) + , CreatedTime(TInstant::Now()) + , ReturnConnectFailedImmediately(false) + , GranStatus(Config.Secret.StatusFlushPeriod) + , QuotaMsg(!Session->IsSource_, Config.PerConnectionMaxInFlight, 0) + , QuotaBytes(!Session->IsSource_, Config.PerConnectionMaxInFlightBySize, 0) + , MaxBufferSize(session->Config.MaxBufferSize) + , ShutdownReason(MESSAGE_OK) + { + WriterData.Status.ConnectionId = connectionId; + WriterData.Status.PeerAddr = PeerAddr; + ReaderData.Status.ConnectionId = connectionId; + + const TInstant now = TInstant::Now(); + + WriterFillStatus(); + + GranStatus.Writer.Update(WriterData.Status, now, true); + GranStatus.Reader.Update(ReaderData.Status, now, true); + } + + TRemoteConnection::~TRemoteConnection() { + Y_VERIFY(ReplyQueue.IsEmpty()); + } + + TRemoteConnection::TWriterData::TWriterData() + : Down(0) + , SocketVersion(0) + , InFlight(0) + , AwakeFlags(0) + , State(WRITER_FILLING) + { + } + + TRemoteConnection::TWriterData::~TWriterData() { Y_VERIFY(AtomicGet(Down)); - Y_VERIFY(SendQueue.Empty()); - } - - bool TRemoteConnection::TReaderData::HasBytesInBuf(size_t bytes) noexcept { - size_t left = Buffer.Size() - Offset; - - return (MoreBytes = left >= bytes ? 0 : bytes - left) == 0; - } - - void TRemoteConnection::TWriterData::SetChannel(NEventLoop::TChannelPtr channel) { - Y_VERIFY(!Channel, "must not have channel"); - Y_VERIFY(Buffer.GetBuffer().Empty() && Buffer.LeftSize() == 0, "buffer must be empty"); - Y_VERIFY(State == WRITER_FILLING, "state must be initial"); - Channel = channel; - } - - void TRemoteConnection::TReaderData::SetChannel(NEventLoop::TChannelPtr channel) { - Y_VERIFY(!Channel, "must not have channel"); - Y_VERIFY(Buffer.Empty(), "buffer must be empty"); - Channel = channel; - } - - void TRemoteConnection::TWriterData::DropChannel() { - if (!!Channel) { - Channel->Unregister(); - Channel.Drop(); - } - - Buffer.Reset(); - State = WRITER_FILLING; - } - - void TRemoteConnection::TReaderData::DropChannel() { - // TODO: make Drop call Unregister - if (!!Channel) { - Channel->Unregister(); - Channel.Drop(); - } - Buffer.Reset(); - Offset = 0; - } - - TRemoteConnection::TReaderData::TReaderData() - : Down(0) - , SocketVersion(0) - , Offset(0) - , MoreBytes(0) - { - } - - TRemoteConnection::TReaderData::~TReaderData() { + Y_VERIFY(SendQueue.Empty()); + } + + bool TRemoteConnection::TReaderData::HasBytesInBuf(size_t bytes) noexcept { + size_t left = Buffer.Size() - Offset; + + return (MoreBytes = left >= bytes ? 0 : bytes - left) == 0; + } + + void TRemoteConnection::TWriterData::SetChannel(NEventLoop::TChannelPtr channel) { + Y_VERIFY(!Channel, "must not have channel"); + Y_VERIFY(Buffer.GetBuffer().Empty() && Buffer.LeftSize() == 0, "buffer must be empty"); + Y_VERIFY(State == WRITER_FILLING, "state must be initial"); + Channel = channel; + } + + void TRemoteConnection::TReaderData::SetChannel(NEventLoop::TChannelPtr channel) { + Y_VERIFY(!Channel, "must not have channel"); + Y_VERIFY(Buffer.Empty(), "buffer must be empty"); + Channel = channel; + } + + void TRemoteConnection::TWriterData::DropChannel() { + if (!!Channel) { + Channel->Unregister(); + Channel.Drop(); + } + + Buffer.Reset(); + State = WRITER_FILLING; + } + + void TRemoteConnection::TReaderData::DropChannel() { + // TODO: make Drop call Unregister + if (!!Channel) { + Channel->Unregister(); + Channel.Drop(); + } + Buffer.Reset(); + Offset = 0; + } + + TRemoteConnection::TReaderData::TReaderData() + : Down(0) + , SocketVersion(0) + , Offset(0) + , MoreBytes(0) + { + } + + TRemoteConnection::TReaderData::~TReaderData() { Y_VERIFY(AtomicGet(Down)); - } + } + + void TRemoteConnection::Send(TNonDestroyingAutoPtr<TBusMessage> msg) { + BeforeSendQueue.Enqueue(msg.Release()); + AtomicIncrement(WriterData.InFlight); + ScheduleWrite(); + } + + void TRemoteConnection::ClearOutgoingQueue(TMessagesPtrs& result, bool reconnect) { + if (!reconnect) { + // Do not clear send queue if reconnecting + WriterData.SendQueue.Clear(&result); + } + } - void TRemoteConnection::Send(TNonDestroyingAutoPtr<TBusMessage> msg) { - BeforeSendQueue.Enqueue(msg.Release()); - AtomicIncrement(WriterData.InFlight); - ScheduleWrite(); - } + void TRemoteConnection::Shutdown(EMessageStatus status) { + ScheduleShutdown(status); - void TRemoteConnection::ClearOutgoingQueue(TMessagesPtrs& result, bool reconnect) { - if (!reconnect) { - // Do not clear send queue if reconnecting - WriterData.SendQueue.Clear(&result); - } - } + ReaderData.ShutdownComplete.WaitI(); + WriterData.ShutdownComplete.WaitI(); + } - void TRemoteConnection::Shutdown(EMessageStatus status) { - ScheduleShutdown(status); + void TRemoteConnection::TryConnect() { + Y_FAIL("TryConnect is client connection only operation"); + } - ReaderData.ShutdownComplete.WaitI(); - WriterData.ShutdownComplete.WaitI(); - } + void TRemoteConnection::ScheduleRead() { + GetReaderActor()->Schedule(); + } - void TRemoteConnection::TryConnect() { - Y_FAIL("TryConnect is client connection only operation"); - } + void TRemoteConnection::ScheduleWrite() { + GetWriterActor()->Schedule(); + } - void TRemoteConnection::ScheduleRead() { - GetReaderActor()->Schedule(); - } + void TRemoteConnection::WriterRotateCounters() { + if (!WriterData.TimeToRotateCounters.FetchTask()) { + return; + } - void TRemoteConnection::ScheduleWrite() { - GetWriterActor()->Schedule(); - } + WriterData.Status.DurationCounterPrev = WriterData.Status.DurationCounter; + Reset(WriterData.Status.DurationCounter); + } - void TRemoteConnection::WriterRotateCounters() { - if (!WriterData.TimeToRotateCounters.FetchTask()) { - return; - } + void TRemoteConnection::WriterSendStatus(TInstant now, bool force) { + GranStatus.Writer.Update(std::bind(&TRemoteConnection::WriterGetStatus, this), now, force); + } - WriterData.Status.DurationCounterPrev = WriterData.Status.DurationCounter; - Reset(WriterData.Status.DurationCounter); - } + void TRemoteConnection::ReaderSendStatus(TInstant now, bool force) { + GranStatus.Reader.Update(std::bind(&TRemoteConnection::ReaderFillStatus, this), now, force); + } - void TRemoteConnection::WriterSendStatus(TInstant now, bool force) { - GranStatus.Writer.Update(std::bind(&TRemoteConnection::WriterGetStatus, this), now, force); - } + const TRemoteConnectionReaderStatus& TRemoteConnection::ReaderFillStatus() { + ReaderData.Status.BufferSize = ReaderData.Buffer.Capacity(); + ReaderData.Status.QuotaMsg = QuotaMsg.Tokens(); + ReaderData.Status.QuotaBytes = QuotaBytes.Tokens(); - void TRemoteConnection::ReaderSendStatus(TInstant now, bool force) { - GranStatus.Reader.Update(std::bind(&TRemoteConnection::ReaderFillStatus, this), now, force); - } - - const TRemoteConnectionReaderStatus& TRemoteConnection::ReaderFillStatus() { - ReaderData.Status.BufferSize = ReaderData.Buffer.Capacity(); - ReaderData.Status.QuotaMsg = QuotaMsg.Tokens(); - ReaderData.Status.QuotaBytes = QuotaBytes.Tokens(); - - return ReaderData.Status; - } + return ReaderData.Status; + } - void TRemoteConnection::ProcessItem(TReaderTag, ::NActor::TDefaultTag, TWriterToReaderSocketMessage readSocket) { - if (AtomicGet(ReaderData.Down)) { - ReaderData.Status.Fd = INVALID_SOCKET; - return; - } + void TRemoteConnection::ProcessItem(TReaderTag, ::NActor::TDefaultTag, TWriterToReaderSocketMessage readSocket) { + if (AtomicGet(ReaderData.Down)) { + ReaderData.Status.Fd = INVALID_SOCKET; + return; + } - ReaderData.DropChannel(); + ReaderData.DropChannel(); - ReaderData.Status.Fd = readSocket.Socket; - ReaderData.SocketVersion = readSocket.SocketVersion; + ReaderData.Status.Fd = readSocket.Socket; + ReaderData.SocketVersion = readSocket.SocketVersion; - if (readSocket.Socket != INVALID_SOCKET) { - ReaderData.SetChannel(Session->ReadEventLoop.Register(readSocket.Socket, this, ReadCookie)); - ReaderData.Channel->EnableRead(); - } - } + if (readSocket.Socket != INVALID_SOCKET) { + ReaderData.SetChannel(Session->ReadEventLoop.Register(readSocket.Socket, this, ReadCookie)); + ReaderData.Channel->EnableRead(); + } + } - void TRemoteConnection::ProcessItem(TWriterTag, TReconnectTag, ui32 socketVersion) { - Y_VERIFY(socketVersion <= WriterData.SocketVersion, "something weird"); + void TRemoteConnection::ProcessItem(TWriterTag, TReconnectTag, ui32 socketVersion) { + Y_VERIFY(socketVersion <= WriterData.SocketVersion, "something weird"); - if (WriterData.SocketVersion != socketVersion) { - return; - } - Y_VERIFY(WriterData.Status.Connected, "must be connected at this point"); - Y_VERIFY(!!WriterData.Channel, "must have channel at this point"); + if (WriterData.SocketVersion != socketVersion) { + return; + } + Y_VERIFY(WriterData.Status.Connected, "must be connected at this point"); + Y_VERIFY(!!WriterData.Channel, "must have channel at this point"); - WriterData.Status.Connected = false; - WriterData.DropChannel(); - WriterData.Status.MyAddr = TNetAddr(); - ++WriterData.SocketVersion; - LastConnectAttempt = TInstant(); + WriterData.Status.Connected = false; + WriterData.DropChannel(); + WriterData.Status.MyAddr = TNetAddr(); + ++WriterData.SocketVersion; + LastConnectAttempt = TInstant(); - TMessagesPtrs cleared; - ClearOutgoingQueue(cleared, true); - WriterErrorMessages(cleared, MESSAGE_DELIVERY_FAILED); + TMessagesPtrs cleared; + ClearOutgoingQueue(cleared, true); + WriterErrorMessages(cleared, MESSAGE_DELIVERY_FAILED); - FireClientConnectionEvent(TClientConnectionEvent::DISCONNECTED); + FireClientConnectionEvent(TClientConnectionEvent::DISCONNECTED); - ReaderGetSocketQueue()->EnqueueAndSchedule(TWriterToReaderSocketMessage(INVALID_SOCKET, WriterData.SocketVersion)); - } + ReaderGetSocketQueue()->EnqueueAndSchedule(TWriterToReaderSocketMessage(INVALID_SOCKET, WriterData.SocketVersion)); + } - void TRemoteConnection::ProcessItem(TWriterTag, TWakeReaderTag, ui32 awakeFlags) { - WriterData.AwakeFlags |= awakeFlags; + void TRemoteConnection::ProcessItem(TWriterTag, TWakeReaderTag, ui32 awakeFlags) { + WriterData.AwakeFlags |= awakeFlags; - ReadQuotaWakeup(); - } + ReadQuotaWakeup(); + } - void TRemoteConnection::Act(TReaderTag) { - TInstant now = TInstant::Now(); + void TRemoteConnection::Act(TReaderTag) { + TInstant now = TInstant::Now(); - ReaderData.Status.Acts += 1; + ReaderData.Status.Acts += 1; - ReaderGetSocketQueue()->DequeueAllLikelyEmpty(); + ReaderGetSocketQueue()->DequeueAllLikelyEmpty(); - if (AtomicGet(ReaderData.Down)) { - ReaderData.DropChannel(); + if (AtomicGet(ReaderData.Down)) { + ReaderData.DropChannel(); - ReaderProcessStatusDown(); - ReaderData.ShutdownComplete.Signal(); + ReaderProcessStatusDown(); + ReaderData.ShutdownComplete.Signal(); - } else if (!!ReaderData.Channel) { - Y_ASSERT(ReaderData.ReadMessages.empty()); + } else if (!!ReaderData.Channel) { + Y_ASSERT(ReaderData.ReadMessages.empty()); - for (int i = 0;; ++i) { - if (i == 100) { - // perform other tasks - GetReaderActor()->AddTaskFromActorLoop(); - break; - } + for (int i = 0;; ++i) { + if (i == 100) { + // perform other tasks + GetReaderActor()->AddTaskFromActorLoop(); + break; + } - if (NeedInterruptRead()) { - ReaderData.Channel->EnableRead(); - break; - } + if (NeedInterruptRead()) { + ReaderData.Channel->EnableRead(); + break; + } - if (!ReaderFillBuffer()) - break; + if (!ReaderFillBuffer()) + break; - if (!ReaderProcessBuffer()) - break; - } + if (!ReaderProcessBuffer()) + break; + } - ReaderFlushMessages(); + ReaderFlushMessages(); } - ReaderSendStatus(now); + ReaderSendStatus(now); } - bool TRemoteConnection::QuotaAcquire(size_t msg, size_t bytes) { - ui32 wakeFlags = 0; + bool TRemoteConnection::QuotaAcquire(size_t msg, size_t bytes) { + ui32 wakeFlags = 0; - if (!QuotaMsg.Acquire(msg)) - wakeFlags |= WAKE_QUOTA_MSG; + if (!QuotaMsg.Acquire(msg)) + wakeFlags |= WAKE_QUOTA_MSG; - else if (!QuotaBytes.Acquire(bytes)) - wakeFlags |= WAKE_QUOTA_BYTES; + else if (!QuotaBytes.Acquire(bytes)) + wakeFlags |= WAKE_QUOTA_BYTES; - if (wakeFlags) { - ReaderData.Status.QuotaExhausted++; + if (wakeFlags) { + ReaderData.Status.QuotaExhausted++; - WriterGetWakeQueue()->EnqueueAndSchedule(wakeFlags); - } + WriterGetWakeQueue()->EnqueueAndSchedule(wakeFlags); + } - return wakeFlags == 0; - } + return wakeFlags == 0; + } - void TRemoteConnection::QuotaConsume(size_t msg, size_t bytes) { - QuotaMsg.Consume(msg); - QuotaBytes.Consume(bytes); - } + void TRemoteConnection::QuotaConsume(size_t msg, size_t bytes) { + QuotaMsg.Consume(msg); + QuotaBytes.Consume(bytes); + } - void TRemoteConnection::QuotaReturnSelf(size_t items, size_t bytes) { - if (QuotaReturnValues(items, bytes)) - ReadQuotaWakeup(); - } + void TRemoteConnection::QuotaReturnSelf(size_t items, size_t bytes) { + if (QuotaReturnValues(items, bytes)) + ReadQuotaWakeup(); + } - void TRemoteConnection::QuotaReturnAside(size_t items, size_t bytes) { - if (QuotaReturnValues(items, bytes) && !AtomicGet(WriterData.Down)) - WriterGetWakeQueue()->EnqueueAndSchedule(0x0); - } + void TRemoteConnection::QuotaReturnAside(size_t items, size_t bytes) { + if (QuotaReturnValues(items, bytes) && !AtomicGet(WriterData.Down)) + WriterGetWakeQueue()->EnqueueAndSchedule(0x0); + } - bool TRemoteConnection::QuotaReturnValues(size_t items, size_t bytes) { - bool rMsg = QuotaMsg.Return(items); - bool rBytes = QuotaBytes.Return(bytes); + bool TRemoteConnection::QuotaReturnValues(size_t items, size_t bytes) { + bool rMsg = QuotaMsg.Return(items); + bool rBytes = QuotaBytes.Return(bytes); - return rMsg || rBytes; - } + return rMsg || rBytes; + } - void TRemoteConnection::ReadQuotaWakeup() { - const ui32 mask = WriterData.AwakeFlags & WriteWakeFlags(); + void TRemoteConnection::ReadQuotaWakeup() { + const ui32 mask = WriterData.AwakeFlags & WriteWakeFlags(); - if (mask && mask == WriterData.AwakeFlags) { - WriterData.Status.ReaderWakeups++; - WriterData.AwakeFlags = 0; + if (mask && mask == WriterData.AwakeFlags) { + WriterData.Status.ReaderWakeups++; + WriterData.AwakeFlags = 0; - ScheduleRead(); - } - } + ScheduleRead(); + } + } - ui32 TRemoteConnection::WriteWakeFlags() const { - ui32 awakeFlags = 0; + ui32 TRemoteConnection::WriteWakeFlags() const { + ui32 awakeFlags = 0; - if (QuotaMsg.IsAboveWake()) - awakeFlags |= WAKE_QUOTA_MSG; + if (QuotaMsg.IsAboveWake()) + awakeFlags |= WAKE_QUOTA_MSG; - if (QuotaBytes.IsAboveWake()) - awakeFlags |= WAKE_QUOTA_BYTES; + if (QuotaBytes.IsAboveWake()) + awakeFlags |= WAKE_QUOTA_BYTES; - return awakeFlags; - } + return awakeFlags; + } - bool TRemoteConnection::ReaderProcessBuffer() { - TInstant now = TInstant::Now(); - - for (;;) { - if (!ReaderData.HasBytesInBuf(sizeof(TBusHeader))) { - break; - } - - TBusHeader header(MakeArrayRef(ReaderData.Buffer.Data() + ReaderData.Offset, ReaderData.Buffer.Size() - ReaderData.Offset)); - - if (header.Size < sizeof(TBusHeader)) { - LWPROBE(Error, ToString(MESSAGE_HEADER_CORRUPTED), ToString(PeerAddr), ToString(header.Size)); - ReaderData.Status.Incremental.StatusCounter[MESSAGE_HEADER_CORRUPTED] += 1; - ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_HEADER_CORRUPTED, false); - return false; - } - - if (!IsVersionNegotiation(header) && !IsBusKeyValid(header.Id)) { - LWPROBE(Error, ToString(MESSAGE_HEADER_CORRUPTED), ToString(PeerAddr), ToString(header.Size)); - ReaderData.Status.Incremental.StatusCounter[MESSAGE_HEADER_CORRUPTED] += 1; - ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_HEADER_CORRUPTED, false); - return false; - } - - if (header.Size > Config.MaxMessageSize) { - LWPROBE(Error, ToString(MESSAGE_MESSAGE_TOO_LARGE), ToString(PeerAddr), ToString(header.Size)); - ReaderData.Status.Incremental.StatusCounter[MESSAGE_MESSAGE_TOO_LARGE] += 1; - ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_MESSAGE_TOO_LARGE, false); - return false; - } - - if (!ReaderData.HasBytesInBuf(header.Size)) { - if (ReaderData.Offset == 0) { - ReaderData.Buffer.Reserve(header.Size); - } - break; - } - - if (!QuotaAcquire(1, header.Size)) - return false; - - if (!MessageRead(MakeArrayRef(ReaderData.Buffer.Data() + ReaderData.Offset, header.Size), now)) { - return false; - } - - ReaderData.Offset += header.Size; - } + bool TRemoteConnection::ReaderProcessBuffer() { + TInstant now = TInstant::Now(); - ReaderData.Buffer.ChopHead(ReaderData.Offset); - ReaderData.Offset = 0; + for (;;) { + if (!ReaderData.HasBytesInBuf(sizeof(TBusHeader))) { + break; + } - if (ReaderData.Buffer.Capacity() > MaxBufferSize && ReaderData.Buffer.Size() <= MaxBufferSize) { - ReaderData.Status.Incremental.BufferDrops += 1; + TBusHeader header(MakeArrayRef(ReaderData.Buffer.Data() + ReaderData.Offset, ReaderData.Buffer.Size() - ReaderData.Offset)); - TBuffer temp; - // probably should use another constant - temp.Reserve(Config.DefaultBufferSize); - temp.Append(ReaderData.Buffer.Data(), ReaderData.Buffer.Size()); + if (header.Size < sizeof(TBusHeader)) { + LWPROBE(Error, ToString(MESSAGE_HEADER_CORRUPTED), ToString(PeerAddr), ToString(header.Size)); + ReaderData.Status.Incremental.StatusCounter[MESSAGE_HEADER_CORRUPTED] += 1; + ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_HEADER_CORRUPTED, false); + return false; + } - ReaderData.Buffer.Swap(temp); - } + if (!IsVersionNegotiation(header) && !IsBusKeyValid(header.Id)) { + LWPROBE(Error, ToString(MESSAGE_HEADER_CORRUPTED), ToString(PeerAddr), ToString(header.Size)); + ReaderData.Status.Incremental.StatusCounter[MESSAGE_HEADER_CORRUPTED] += 1; + ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_HEADER_CORRUPTED, false); + return false; + } - return true; - } + if (header.Size > Config.MaxMessageSize) { + LWPROBE(Error, ToString(MESSAGE_MESSAGE_TOO_LARGE), ToString(PeerAddr), ToString(header.Size)); + ReaderData.Status.Incremental.StatusCounter[MESSAGE_MESSAGE_TOO_LARGE] += 1; + ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_MESSAGE_TOO_LARGE, false); + return false; + } - bool TRemoteConnection::ReaderFillBuffer() { - if (!ReaderData.BufferMore()) - return true; + if (!ReaderData.HasBytesInBuf(header.Size)) { + if (ReaderData.Offset == 0) { + ReaderData.Buffer.Reserve(header.Size); + } + break; + } - if (ReaderData.Buffer.Avail() == 0) { - if (ReaderData.Buffer.Size() == 0) { - ReaderData.Buffer.Reserve(Config.DefaultBufferSize); - } else { - ReaderData.Buffer.Reserve(ReaderData.Buffer.Size() * 2); - } - } + if (!QuotaAcquire(1, header.Size)) + return false; - Y_ASSERT(ReaderData.Buffer.Avail() > 0); + if (!MessageRead(MakeArrayRef(ReaderData.Buffer.Data() + ReaderData.Offset, header.Size), now)) { + return false; + } - ssize_t bytes; - { - TWhatThreadDoesPushPop pp("recv syscall"); - bytes = SocketRecv(ReaderData.Channel->GetSocket(), TArrayRef<char>(ReaderData.Buffer.Pos(), ReaderData.Buffer.Avail())); - } + ReaderData.Offset += header.Size; + } - if (bytes < 0) { - if (WouldBlock()) { - ReaderData.Channel->EnableRead(); - return false; - } else { - ReaderData.Channel->DisableRead(); - ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_DELIVERY_FAILED, false); - return false; - } - } + ReaderData.Buffer.ChopHead(ReaderData.Offset); + ReaderData.Offset = 0; - if (bytes == 0) { - ReaderData.Channel->DisableRead(); - // TODO: incorrect: it is possible that only input is shutdown, and output is available - ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_DELIVERY_FAILED, false); - return false; - } + if (ReaderData.Buffer.Capacity() > MaxBufferSize && ReaderData.Buffer.Size() <= MaxBufferSize) { + ReaderData.Status.Incremental.BufferDrops += 1; - ReaderData.Status.Incremental.NetworkOps += 1; + TBuffer temp; + // probably should use another constant + temp.Reserve(Config.DefaultBufferSize); + temp.Append(ReaderData.Buffer.Data(), ReaderData.Buffer.Size()); - ReaderData.Buffer.Advance(bytes); - ReaderData.MoreBytes = 0; - return true; - } + ReaderData.Buffer.Swap(temp); + } - void TRemoteConnection::ClearBeforeSendQueue(EMessageStatus reason) { - BeforeSendQueue.DequeueAll(std::bind(&TRemoteConnection::WriterBeforeWriteErrorMessage, this, std::placeholders::_1, reason)); + return true; } - void TRemoteConnection::ClearReplyQueue(EMessageStatus reason) { - TVectorSwaps<TBusMessagePtrAndHeader> replyQueueTemp; - Y_ASSERT(replyQueueTemp.empty()); - ReplyQueue.DequeueAllSingleConsumer(&replyQueueTemp); + bool TRemoteConnection::ReaderFillBuffer() { + if (!ReaderData.BufferMore()) + return true; - TVector<TBusMessage*> messages; - for (TVectorSwaps<TBusMessagePtrAndHeader>::reverse_iterator message = replyQueueTemp.rbegin(); - message != replyQueueTemp.rend(); ++message) { - messages.push_back(message->MessagePtr.Release()); + if (ReaderData.Buffer.Avail() == 0) { + if (ReaderData.Buffer.Size() == 0) { + ReaderData.Buffer.Reserve(Config.DefaultBufferSize); + } else { + ReaderData.Buffer.Reserve(ReaderData.Buffer.Size() * 2); + } } - WriterErrorMessages(messages, reason); + Y_ASSERT(ReaderData.Buffer.Avail() > 0); - replyQueueTemp.clear(); - } + ssize_t bytes; + { + TWhatThreadDoesPushPop pp("recv syscall"); + bytes = SocketRecv(ReaderData.Channel->GetSocket(), TArrayRef<char>(ReaderData.Buffer.Pos(), ReaderData.Buffer.Avail())); + } + + if (bytes < 0) { + if (WouldBlock()) { + ReaderData.Channel->EnableRead(); + return false; + } else { + ReaderData.Channel->DisableRead(); + ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_DELIVERY_FAILED, false); + return false; + } + } + + if (bytes == 0) { + ReaderData.Channel->DisableRead(); + // TODO: incorrect: it is possible that only input is shutdown, and output is available + ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_DELIVERY_FAILED, false); + return false; + } + + ReaderData.Status.Incremental.NetworkOps += 1; + + ReaderData.Buffer.Advance(bytes); + ReaderData.MoreBytes = 0; + return true; + } + + void TRemoteConnection::ClearBeforeSendQueue(EMessageStatus reason) { + BeforeSendQueue.DequeueAll(std::bind(&TRemoteConnection::WriterBeforeWriteErrorMessage, this, std::placeholders::_1, reason)); + } + + void TRemoteConnection::ClearReplyQueue(EMessageStatus reason) { + TVectorSwaps<TBusMessagePtrAndHeader> replyQueueTemp; + Y_ASSERT(replyQueueTemp.empty()); + ReplyQueue.DequeueAllSingleConsumer(&replyQueueTemp); + + TVector<TBusMessage*> messages; + for (TVectorSwaps<TBusMessagePtrAndHeader>::reverse_iterator message = replyQueueTemp.rbegin(); + message != replyQueueTemp.rend(); ++message) { + messages.push_back(message->MessagePtr.Release()); + } + + WriterErrorMessages(messages, reason); + + replyQueueTemp.clear(); + } + + void TRemoteConnection::ProcessBeforeSendQueueMessage(TBusMessage* message, TInstant now) { + // legacy clients expect this field to be set + if (!Session->IsSource_) { + message->SendTime = now.MilliSeconds(); + } + + WriterData.SendQueue.PushBack(message); + } + + void TRemoteConnection::ProcessBeforeSendQueue(TInstant now) { + BeforeSendQueue.DequeueAll(std::bind(&TRemoteConnection::ProcessBeforeSendQueueMessage, this, std::placeholders::_1, now)); + } + + void TRemoteConnection::WriterFillInFlight() { + // this is hack for TLoadBalancedProtocol + WriterFillStatus(); + AtomicSet(WriterData.InFlight, WriterData.Status.GetInFlight()); + } + + const TRemoteConnectionWriterStatus& TRemoteConnection::WriterGetStatus() { + WriterRotateCounters(); + WriterFillStatus(); + + return WriterData.Status; + } + + void TRemoteConnection::WriterFillStatus() { + if (!!WriterData.Channel) { + WriterData.Status.Fd = WriterData.Channel->GetSocket(); + } else { + WriterData.Status.Fd = INVALID_SOCKET; + } + WriterData.Status.BufferSize = WriterData.Buffer.Capacity(); + WriterData.Status.SendQueueSize = WriterData.SendQueue.Size(); + WriterData.Status.State = WriterData.State; + } + + void TRemoteConnection::WriterProcessStatusDown() { + Session->GetDeadConnectionWriterStatusQueue()->EnqueueAndSchedule(WriterData.Status.Incremental); + Reset(WriterData.Status.Incremental); + } + + void TRemoteConnection::ReaderProcessStatusDown() { + Session->GetDeadConnectionReaderStatusQueue()->EnqueueAndSchedule(ReaderData.Status.Incremental); + Reset(ReaderData.Status.Incremental); + } + + void TRemoteConnection::ProcessWriterDown() { + if (!RemovedFromSession) { + Session->GetRemoveConnectionQueue()->EnqueueAndSchedule(this); + + if (Session->IsSource_) { + if (WriterData.Status.Connected) { + FireClientConnectionEvent(TClientConnectionEvent::DISCONNECTED); + } + } + + LWPROBE(Disconnected, ToString(PeerAddr)); + RemovedFromSession = true; + } + + WriterData.DropChannel(); - void TRemoteConnection::ProcessBeforeSendQueueMessage(TBusMessage* message, TInstant now) { - // legacy clients expect this field to be set - if (!Session->IsSource_) { - message->SendTime = now.MilliSeconds(); - } + DropEnqueuedData(ShutdownReason, MESSAGE_SHUTDOWN); - WriterData.SendQueue.PushBack(message); - } + WriterProcessStatusDown(); - void TRemoteConnection::ProcessBeforeSendQueue(TInstant now) { - BeforeSendQueue.DequeueAll(std::bind(&TRemoteConnection::ProcessBeforeSendQueueMessage, this, std::placeholders::_1, now)); - } + WriterData.ShutdownComplete.Signal(); + } - void TRemoteConnection::WriterFillInFlight() { - // this is hack for TLoadBalancedProtocol - WriterFillStatus(); - AtomicSet(WriterData.InFlight, WriterData.Status.GetInFlight()); - } + void TRemoteConnection::DropEnqueuedData(EMessageStatus reason, EMessageStatus reasonForQueues) { + ClearReplyQueue(reasonForQueues); + ClearBeforeSendQueue(reasonForQueues); + WriterGetReconnectQueue()->Clear(); + WriterGetWakeQueue()->Clear(); - const TRemoteConnectionWriterStatus& TRemoteConnection::WriterGetStatus() { - WriterRotateCounters(); - WriterFillStatus(); + TMessagesPtrs cleared; + ClearOutgoingQueue(cleared, false); - return WriterData.Status; - } + if (!Session->IsSource_) { + for (auto& i : cleared) { + TBusMessagePtrAndHeader h(i); + CheckedCast<TRemoteServerSession*>(Session.Get())->ReleaseInWorkResponses(MakeArrayRef(&h, 1)); + // assignment back is weird + i = h.MessagePtr.Release(); + // and this part is not batch + } + } - void TRemoteConnection::WriterFillStatus() { - if (!!WriterData.Channel) { - WriterData.Status.Fd = WriterData.Channel->GetSocket(); - } else { - WriterData.Status.Fd = INVALID_SOCKET; - } - WriterData.Status.BufferSize = WriterData.Buffer.Capacity(); - WriterData.Status.SendQueueSize = WriterData.SendQueue.Size(); - WriterData.Status.State = WriterData.State; - } + WriterErrorMessages(cleared, reason); + } - void TRemoteConnection::WriterProcessStatusDown() { - Session->GetDeadConnectionWriterStatusQueue()->EnqueueAndSchedule(WriterData.Status.Incremental); - Reset(WriterData.Status.Incremental); - } + void TRemoteConnection::BeforeTryWrite() { + } - void TRemoteConnection::ReaderProcessStatusDown() { - Session->GetDeadConnectionReaderStatusQueue()->EnqueueAndSchedule(ReaderData.Status.Incremental); - Reset(ReaderData.Status.Incremental); - } + void TRemoteConnection::Act(TWriterTag) { + TInstant now = TInstant::Now(); - void TRemoteConnection::ProcessWriterDown() { - if (!RemovedFromSession) { - Session->GetRemoveConnectionQueue()->EnqueueAndSchedule(this); + WriterData.Status.Acts += 1; - if (Session->IsSource_) { - if (WriterData.Status.Connected) { - FireClientConnectionEvent(TClientConnectionEvent::DISCONNECTED); - } - } - - LWPROBE(Disconnected, ToString(PeerAddr)); - RemovedFromSession = true; + if (Y_UNLIKELY(AtomicGet(WriterData.Down))) { + // dump status must work even if WriterDown + WriterSendStatus(now, true); + ProcessWriterDown(); + return; } - WriterData.DropChannel(); + ProcessBeforeSendQueue(now); - DropEnqueuedData(ShutdownReason, MESSAGE_SHUTDOWN); + BeforeTryWrite(); - WriterProcessStatusDown(); + WriterFillInFlight(); - WriterData.ShutdownComplete.Signal(); - } + WriterGetReconnectQueue()->DequeueAllLikelyEmpty(); - void TRemoteConnection::DropEnqueuedData(EMessageStatus reason, EMessageStatus reasonForQueues) { - ClearReplyQueue(reasonForQueues); - ClearBeforeSendQueue(reasonForQueues); - WriterGetReconnectQueue()->Clear(); - WriterGetWakeQueue()->Clear(); - - TMessagesPtrs cleared; - ClearOutgoingQueue(cleared, false); - - if (!Session->IsSource_) { - for (auto& i : cleared) { - TBusMessagePtrAndHeader h(i); - CheckedCast<TRemoteServerSession*>(Session.Get())->ReleaseInWorkResponses(MakeArrayRef(&h, 1)); - // assignment back is weird - i = h.MessagePtr.Release(); - // and this part is not batch - } - } + if (!WriterData.Status.Connected) { + TryConnect(); + } else { + for (int i = 0;; ++i) { + if (i == 100) { + // perform other tasks + GetWriterActor()->AddTaskFromActorLoop(); + break; + } - WriterErrorMessages(cleared, reason); - } + if (WriterData.State == WRITER_FILLING) { + WriterFillBuffer(); - void TRemoteConnection::BeforeTryWrite() { - } - - void TRemoteConnection::Act(TWriterTag) { - TInstant now = TInstant::Now(); - - WriterData.Status.Acts += 1; - - if (Y_UNLIKELY(AtomicGet(WriterData.Down))) { - // dump status must work even if WriterDown - WriterSendStatus(now, true); - ProcessWriterDown(); - return; - } + if (WriterData.State == WRITER_FILLING) { + WriterData.Channel->DisableWrite(); + break; + } - ProcessBeforeSendQueue(now); + Y_ASSERT(!WriterData.Buffer.Empty()); + } - BeforeTryWrite(); + if (WriterData.State == WRITER_FLUSHING) { + WriterFlushBuffer(); - WriterFillInFlight(); + if (WriterData.State == WRITER_FLUSHING) { + break; + } + } + } + } - WriterGetReconnectQueue()->DequeueAllLikelyEmpty(); + WriterGetWakeQueue()->DequeueAllLikelyEmpty(); - if (!WriterData.Status.Connected) { - TryConnect(); - } else { - for (int i = 0;; ++i) { - if (i == 100) { - // perform other tasks - GetWriterActor()->AddTaskFromActorLoop(); - break; - } + WriterSendStatus(now); + } - if (WriterData.State == WRITER_FILLING) { - WriterFillBuffer(); - - if (WriterData.State == WRITER_FILLING) { - WriterData.Channel->DisableWrite(); - break; - } - - Y_ASSERT(!WriterData.Buffer.Empty()); - } - - if (WriterData.State == WRITER_FLUSHING) { - WriterFlushBuffer(); - - if (WriterData.State == WRITER_FLUSHING) { - break; - } - } - } - } + void TRemoteConnection::WriterFlushBuffer() { + Y_ASSERT(WriterData.State == WRITER_FLUSHING); + Y_ASSERT(!WriterData.Buffer.Empty()); - WriterGetWakeQueue()->DequeueAllLikelyEmpty(); + WriterData.CorkUntil = TInstant::Zero(); - WriterSendStatus(now); - } + while (!WriterData.Buffer.Empty()) { + ssize_t bytes; + { + TWhatThreadDoesPushPop pp("send syscall"); + bytes = SocketSend(WriterData.Channel->GetSocket(), TArrayRef<const char>(WriterData.Buffer.LeftPos(), WriterData.Buffer.Size())); + } - void TRemoteConnection::WriterFlushBuffer() { - Y_ASSERT(WriterData.State == WRITER_FLUSHING); - Y_ASSERT(!WriterData.Buffer.Empty()); + if (bytes < 0) { + if (WouldBlock()) { + WriterData.Channel->EnableWrite(); + return; + } else { + WriterData.Channel->DisableWrite(); + ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_DELIVERY_FAILED, true); + return; + } + } - WriterData.CorkUntil = TInstant::Zero(); + WriterData.Status.Incremental.NetworkOps += 1; - while (!WriterData.Buffer.Empty()) { - ssize_t bytes; - { - TWhatThreadDoesPushPop pp("send syscall"); - bytes = SocketSend(WriterData.Channel->GetSocket(), TArrayRef<const char>(WriterData.Buffer.LeftPos(), WriterData.Buffer.Size())); - } - - if (bytes < 0) { - if (WouldBlock()) { - WriterData.Channel->EnableWrite(); - return; - } else { - WriterData.Channel->DisableWrite(); - ScheduleShutdownOnServerOrReconnectOnClient(MESSAGE_DELIVERY_FAILED, true); - return; - } - } - - WriterData.Status.Incremental.NetworkOps += 1; - - WriterData.Buffer.LeftProceed(bytes); + WriterData.Buffer.LeftProceed(bytes); } - WriterData.Buffer.Clear(); - if (WriterData.Buffer.Capacity() > MaxBufferSize) { - WriterData.Status.Incremental.BufferDrops += 1; - WriterData.Buffer.Reset(); + WriterData.Buffer.Clear(); + if (WriterData.Buffer.Capacity() > MaxBufferSize) { + WriterData.Status.Incremental.BufferDrops += 1; + WriterData.Buffer.Reset(); } - WriterData.State = WRITER_FILLING; - } + WriterData.State = WRITER_FILLING; + } - void TRemoteConnection::ScheduleShutdownOnServerOrReconnectOnClient(EMessageStatus status, bool writer) { - if (Session->IsSource_) { - WriterGetReconnectQueue()->EnqueueAndSchedule(writer ? WriterData.SocketVersion : ReaderData.SocketVersion); - } else { - ScheduleShutdown(status); + void TRemoteConnection::ScheduleShutdownOnServerOrReconnectOnClient(EMessageStatus status, bool writer) { + if (Session->IsSource_) { + WriterGetReconnectQueue()->EnqueueAndSchedule(writer ? WriterData.SocketVersion : ReaderData.SocketVersion); + } else { + ScheduleShutdown(status); } } - void TRemoteConnection::ScheduleShutdown(EMessageStatus status) { - ShutdownReason = status; + void TRemoteConnection::ScheduleShutdown(EMessageStatus status) { + ShutdownReason = status; - AtomicSet(ReaderData.Down, 1); - ScheduleRead(); + AtomicSet(ReaderData.Down, 1); + ScheduleRead(); - AtomicSet(WriterData.Down, 1); - ScheduleWrite(); - } + AtomicSet(WriterData.Down, 1); + ScheduleWrite(); + } - void TRemoteConnection::CallSerialize(TBusMessage* msg, TBuffer& buffer) const { - size_t posForAssertion = buffer.Size(); - Proto->Serialize(msg, buffer); - Y_VERIFY(buffer.Size() >= posForAssertion, - "incorrect Serialize implementation, pos before serialize: %d, pos after serialize: %d", - int(posForAssertion), int(buffer.Size())); + void TRemoteConnection::CallSerialize(TBusMessage* msg, TBuffer& buffer) const { + size_t posForAssertion = buffer.Size(); + Proto->Serialize(msg, buffer); + Y_VERIFY(buffer.Size() >= posForAssertion, + "incorrect Serialize implementation, pos before serialize: %d, pos after serialize: %d", + int(posForAssertion), int(buffer.Size())); } - namespace { + namespace { inline void WriteHeader(const TBusHeader& header, TBuffer& data) { - data.Reserve(data.Size() + sizeof(TBusHeader)); - /// \todo hton instead of memcpy - memcpy(data.Data() + data.Size(), &header, sizeof(TBusHeader)); - data.Advance(sizeof(TBusHeader)); + data.Reserve(data.Size() + sizeof(TBusHeader)); + /// \todo hton instead of memcpy + memcpy(data.Data() + data.Size(), &header, sizeof(TBusHeader)); + data.Advance(sizeof(TBusHeader)); } inline void WriteDummyHeader(TBuffer& data) { - data.Resize(data.Size() + sizeof(TBusHeader)); - } + data.Resize(data.Size() + sizeof(TBusHeader)); + } - } + } - void TRemoteConnection::SerializeMessage(TBusMessage* msg, TBuffer* data, TMessageCounter* counter) const { - size_t pos = data->Size(); + void TRemoteConnection::SerializeMessage(TBusMessage* msg, TBuffer* data, TMessageCounter* counter) const { + size_t pos = data->Size(); - size_t dataSize; + size_t dataSize; - bool compressionRequested = msg->IsCompressed(); + bool compressionRequested = msg->IsCompressed(); - if (compressionRequested) { - TBuffer compdata; - TBuffer plaindata; - CallSerialize(msg, plaindata); + if (compressionRequested) { + TBuffer compdata; + TBuffer plaindata; + CallSerialize(msg, plaindata); - dataSize = sizeof(TBusHeader) + plaindata.Size(); + dataSize = sizeof(TBusHeader) + plaindata.Size(); - NCodecs::TCodecPtr c = Proto->GetTransportCodec(); + NCodecs::TCodecPtr c = Proto->GetTransportCodec(); c->Encode(TStringBuf{plaindata.data(), plaindata.size()}, compdata); - if (compdata.Size() < plaindata.Size()) { - plaindata.Clear(); - msg->GetHeader()->Size = sizeof(TBusHeader) + compdata.Size(); - WriteHeader(*msg->GetHeader(), *data); - data->Append(compdata.Data(), compdata.Size()); - } else { - compdata.Clear(); - msg->SetCompressed(false); - msg->GetHeader()->Size = sizeof(TBusHeader) + plaindata.Size(); - WriteHeader(*msg->GetHeader(), *data); - data->Append(plaindata.Data(), plaindata.Size()); - } - } else { - WriteDummyHeader(*data); - CallSerialize(msg, *data); - - dataSize = msg->GetHeader()->Size = data->Size() - pos; - - data->Proceed(pos); - WriteHeader(*msg->GetHeader(), *data); - data->Proceed(pos + msg->GetHeader()->Size); - } + if (compdata.Size() < plaindata.Size()) { + plaindata.Clear(); + msg->GetHeader()->Size = sizeof(TBusHeader) + compdata.Size(); + WriteHeader(*msg->GetHeader(), *data); + data->Append(compdata.Data(), compdata.Size()); + } else { + compdata.Clear(); + msg->SetCompressed(false); + msg->GetHeader()->Size = sizeof(TBusHeader) + plaindata.Size(); + WriteHeader(*msg->GetHeader(), *data); + data->Append(plaindata.Data(), plaindata.Size()); + } + } else { + WriteDummyHeader(*data); + CallSerialize(msg, *data); + + dataSize = msg->GetHeader()->Size = data->Size() - pos; + + data->Proceed(pos); + WriteHeader(*msg->GetHeader(), *data); + data->Proceed(pos + msg->GetHeader()->Size); + } + + Y_ASSERT(msg->GetHeader()->Size == data->Size() - pos); + counter->AddMessage(dataSize, data->Size() - pos, msg->IsCompressed(), compressionRequested); + } + + TBusMessage* TRemoteConnection::DeserializeMessage(TArrayRef<const char> dataRef, const TBusHeader* header, TMessageCounter* messageCounter, EMessageStatus* status) const { + size_t dataSize; + + TBusMessage* message; + if (header->FlagsInternal & MESSAGE_COMPRESS_INTERNAL) { + TBuffer msg; + { + TBuffer plaindata; + NCodecs::TCodecPtr c = Proto->GetTransportCodec(); + try { + TArrayRef<const char> payload = TBusMessage::GetPayload(dataRef); + c->Decode(TStringBuf{payload.data(), payload.size()}, plaindata); + } catch (...) { + // catch all, because + // http://nga.at.yandex-team.ru/replies.xml?item_no=3884 + *status = MESSAGE_DECOMPRESS_ERROR; + return nullptr; + } + + msg.Append(dataRef.data(), sizeof(TBusHeader)); + msg.Append(plaindata.Data(), plaindata.Size()); + } + TArrayRef<const char> msgRef(msg.Data(), msg.Size()); + dataSize = sizeof(TBusHeader) + msgRef.size(); + // TODO: track error types + message = Proto->Deserialize(header->Type, msgRef.Slice(sizeof(TBusHeader))).Release(); + if (!message) { + *status = MESSAGE_DESERIALIZE_ERROR; + return nullptr; + } + *message->GetHeader() = *header; + message->SetCompressed(true); + } else { + dataSize = dataRef.size(); + message = Proto->Deserialize(header->Type, dataRef.Slice(sizeof(TBusHeader))).Release(); + if (!message) { + *status = MESSAGE_DESERIALIZE_ERROR; + return nullptr; + } + *message->GetHeader() = *header; + } + + messageCounter->AddMessage(dataSize, dataRef.size(), header->FlagsInternal & MESSAGE_COMPRESS_INTERNAL, false); + + return message; + } + + void TRemoteConnection::ResetOneWayFlag(TArrayRef<TBusMessage*> messages) { + for (auto message : messages) { + message->LocalFlags &= ~MESSAGE_ONE_WAY_INTERNAL; + } + } + + void TRemoteConnection::ReaderFlushMessages() { + if (!ReaderData.ReadMessages.empty()) { + Session->OnMessageReceived(this, ReaderData.ReadMessages); + ReaderData.ReadMessages.clear(); + } + } + + // @return false if actor should break + bool TRemoteConnection::MessageRead(TArrayRef<const char> readDataRef, TInstant now) { + TBusHeader header(readDataRef); + + Y_ASSERT(readDataRef.size() == header.Size); + + if (header.GetVersionInternal() != YBUS_VERSION) { + ReaderProcessMessageUnknownVersion(readDataRef); + return true; + } + + EMessageStatus deserializeFailureStatus = MESSAGE_OK; + TBusMessage* r = DeserializeMessage(readDataRef, &header, &ReaderData.Status.Incremental.MessageCounter, &deserializeFailureStatus); + + if (!r) { + Y_VERIFY(deserializeFailureStatus != MESSAGE_OK, "state check"); + LWPROBE(Error, ToString(deserializeFailureStatus), ToString(PeerAddr), ""); + ReaderData.Status.Incremental.StatusCounter[deserializeFailureStatus] += 1; + ScheduleShutdownOnServerOrReconnectOnClient(deserializeFailureStatus, false); + return false; + } + + LWPROBE(Read, r->GetHeader()->Size); + + r->ReplyTo = PeerAddrSocketAddr; + + TBusMessagePtrAndHeader h(r); + r->RecvTime = now; + + QuotaConsume(1, header.Size); + + ReaderData.ReadMessages.push_back(h); + if (ReaderData.ReadMessages.size() >= 100) { + ReaderFlushMessages(); + } + + return true; + } + + void TRemoteConnection::WriterFillBuffer() { + Y_ASSERT(WriterData.State == WRITER_FILLING); + + Y_ASSERT(WriterData.Buffer.LeftSize() == 0); + + if (Y_UNLIKELY(!WrongVersionRequests.IsEmpty())) { + TVector<TBusHeader> headers; + WrongVersionRequests.DequeueAllSingleConsumer(&headers); + for (TVector<TBusHeader>::reverse_iterator header = headers.rbegin(); + header != headers.rend(); ++header) { + TBusHeader response = *header; + response.SendTime = NBus::Now(); + response.Size = sizeof(TBusHeader); + response.FlagsInternal = 0; + response.SetVersionInternal(YBUS_VERSION); + WriteHeader(response, WriterData.Buffer.GetBuffer()); + } + + Y_ASSERT(!WriterData.Buffer.Empty()); + WriterData.State = WRITER_FLUSHING; + return; + } + + TTempTlsVector<TBusMessagePtrAndHeader, void, TVectorSwaps> writeMessages; + + for (;;) { + THolder<TBusMessage> writeMessage(WriterData.SendQueue.PopFront()); + if (!writeMessage) { + break; + } + + if (Config.Cork != TDuration::Zero()) { + if (WriterData.CorkUntil == TInstant::Zero()) { + WriterData.CorkUntil = TInstant::Now() + Config.Cork; + } + } - Y_ASSERT(msg->GetHeader()->Size == data->Size() - pos); - counter->AddMessage(dataSize, data->Size() - pos, msg->IsCompressed(), compressionRequested); - } + size_t sizeBeforeSerialize = WriterData.Buffer.Size(); - TBusMessage* TRemoteConnection::DeserializeMessage(TArrayRef<const char> dataRef, const TBusHeader* header, TMessageCounter* messageCounter, EMessageStatus* status) const { - size_t dataSize; - - TBusMessage* message; - if (header->FlagsInternal & MESSAGE_COMPRESS_INTERNAL) { - TBuffer msg; - { - TBuffer plaindata; - NCodecs::TCodecPtr c = Proto->GetTransportCodec(); - try { - TArrayRef<const char> payload = TBusMessage::GetPayload(dataRef); - c->Decode(TStringBuf{payload.data(), payload.size()}, plaindata); - } catch (...) { - // catch all, because - // http://nga.at.yandex-team.ru/replies.xml?item_no=3884 - *status = MESSAGE_DECOMPRESS_ERROR; - return nullptr; - } - - msg.Append(dataRef.data(), sizeof(TBusHeader)); - msg.Append(plaindata.Data(), plaindata.Size()); - } - TArrayRef<const char> msgRef(msg.Data(), msg.Size()); - dataSize = sizeof(TBusHeader) + msgRef.size(); - // TODO: track error types - message = Proto->Deserialize(header->Type, msgRef.Slice(sizeof(TBusHeader))).Release(); - if (!message) { - *status = MESSAGE_DESERIALIZE_ERROR; - return nullptr; - } - *message->GetHeader() = *header; - message->SetCompressed(true); - } else { - dataSize = dataRef.size(); - message = Proto->Deserialize(header->Type, dataRef.Slice(sizeof(TBusHeader))).Release(); - if (!message) { - *status = MESSAGE_DESERIALIZE_ERROR; - return nullptr; - } - *message->GetHeader() = *header; - } + TMessageCounter messageCounter = WriterData.Status.Incremental.MessageCounter; - messageCounter->AddMessage(dataSize, dataRef.size(), header->FlagsInternal & MESSAGE_COMPRESS_INTERNAL, false); + SerializeMessage(writeMessage.Get(), &WriterData.Buffer.GetBuffer(), &messageCounter); - return message; - } - - void TRemoteConnection::ResetOneWayFlag(TArrayRef<TBusMessage*> messages) { - for (auto message : messages) { - message->LocalFlags &= ~MESSAGE_ONE_WAY_INTERNAL; - } - } - - void TRemoteConnection::ReaderFlushMessages() { - if (!ReaderData.ReadMessages.empty()) { - Session->OnMessageReceived(this, ReaderData.ReadMessages); - ReaderData.ReadMessages.clear(); - } - } - - // @return false if actor should break - bool TRemoteConnection::MessageRead(TArrayRef<const char> readDataRef, TInstant now) { - TBusHeader header(readDataRef); + size_t written = WriterData.Buffer.Size() - sizeBeforeSerialize; + if (written > Config.MaxMessageSize) { + WriterData.Buffer.GetBuffer().EraseBack(written); + WriterBeforeWriteErrorMessage(writeMessage.Release(), MESSAGE_MESSAGE_TOO_LARGE); + continue; + } - Y_ASSERT(readDataRef.size() == header.Size); - - if (header.GetVersionInternal() != YBUS_VERSION) { - ReaderProcessMessageUnknownVersion(readDataRef); - return true; - } + WriterData.Status.Incremental.MessageCounter = messageCounter; - EMessageStatus deserializeFailureStatus = MESSAGE_OK; - TBusMessage* r = DeserializeMessage(readDataRef, &header, &ReaderData.Status.Incremental.MessageCounter, &deserializeFailureStatus); + TBusMessagePtrAndHeader h(writeMessage.Release()); + writeMessages.GetVector()->push_back(h); - if (!r) { - Y_VERIFY(deserializeFailureStatus != MESSAGE_OK, "state check"); - LWPROBE(Error, ToString(deserializeFailureStatus), ToString(PeerAddr), ""); - ReaderData.Status.Incremental.StatusCounter[deserializeFailureStatus] += 1; - ScheduleShutdownOnServerOrReconnectOnClient(deserializeFailureStatus, false); - return false; + Y_ASSERT(!WriterData.Buffer.Empty()); + if (WriterData.Buffer.Size() >= Config.SendThreshold) { + break; + } } - - LWPROBE(Read, r->GetHeader()->Size); - - r->ReplyTo = PeerAddrSocketAddr; - - TBusMessagePtrAndHeader h(r); - r->RecvTime = now; - - QuotaConsume(1, header.Size); - - ReaderData.ReadMessages.push_back(h); - if (ReaderData.ReadMessages.size() >= 100) { - ReaderFlushMessages(); - } - - return true; - } - - void TRemoteConnection::WriterFillBuffer() { - Y_ASSERT(WriterData.State == WRITER_FILLING); - - Y_ASSERT(WriterData.Buffer.LeftSize() == 0); - - if (Y_UNLIKELY(!WrongVersionRequests.IsEmpty())) { - TVector<TBusHeader> headers; - WrongVersionRequests.DequeueAllSingleConsumer(&headers); - for (TVector<TBusHeader>::reverse_iterator header = headers.rbegin(); - header != headers.rend(); ++header) { - TBusHeader response = *header; - response.SendTime = NBus::Now(); - response.Size = sizeof(TBusHeader); - response.FlagsInternal = 0; - response.SetVersionInternal(YBUS_VERSION); - WriteHeader(response, WriterData.Buffer.GetBuffer()); - } - - Y_ASSERT(!WriterData.Buffer.Empty()); - WriterData.State = WRITER_FLUSHING; - return; - } - - TTempTlsVector<TBusMessagePtrAndHeader, void, TVectorSwaps> writeMessages; - - for (;;) { - THolder<TBusMessage> writeMessage(WriterData.SendQueue.PopFront()); - if (!writeMessage) { - break; - } - - if (Config.Cork != TDuration::Zero()) { - if (WriterData.CorkUntil == TInstant::Zero()) { - WriterData.CorkUntil = TInstant::Now() + Config.Cork; - } - } - - size_t sizeBeforeSerialize = WriterData.Buffer.Size(); - - TMessageCounter messageCounter = WriterData.Status.Incremental.MessageCounter; - - SerializeMessage(writeMessage.Get(), &WriterData.Buffer.GetBuffer(), &messageCounter); - - size_t written = WriterData.Buffer.Size() - sizeBeforeSerialize; - if (written > Config.MaxMessageSize) { - WriterData.Buffer.GetBuffer().EraseBack(written); - WriterBeforeWriteErrorMessage(writeMessage.Release(), MESSAGE_MESSAGE_TOO_LARGE); - continue; - } - - WriterData.Status.Incremental.MessageCounter = messageCounter; - - TBusMessagePtrAndHeader h(writeMessage.Release()); - writeMessages.GetVector()->push_back(h); - - Y_ASSERT(!WriterData.Buffer.Empty()); - if (WriterData.Buffer.Size() >= Config.SendThreshold) { - break; - } - } - - if (!WriterData.Buffer.Empty()) { - if (WriterData.Buffer.Size() >= Config.SendThreshold) { - WriterData.State = WRITER_FLUSHING; - } else if (WriterData.CorkUntil == TInstant::Zero()) { - WriterData.State = WRITER_FLUSHING; - } else if (TInstant::Now() >= WriterData.CorkUntil) { - WriterData.State = WRITER_FLUSHING; - } else { - // keep filling - Y_ASSERT(WriterData.State == WRITER_FILLING); - GetWriterSchedulerActor()->ScheduleAt(WriterData.CorkUntil); - } - } else { - // keep filling - Y_ASSERT(WriterData.State == WRITER_FILLING); - } - - size_t bytes = MessageSize(*writeMessages.GetVector()); - - QuotaReturnSelf(writeMessages.GetVector()->size(), bytes); - - // This is called before `send` syscall inducing latency - MessageSent(*writeMessages.GetVector()); - } - - size_t TRemoteConnection::MessageSize(TArrayRef<TBusMessagePtrAndHeader> messages) { - size_t size = 0; - for (const auto& message : messages) - size += message.MessagePtr->RequestSize; - - return size; - } - - size_t TRemoteConnection::GetInFlight() { - return AtomicGet(WriterData.InFlight); - } - - size_t TRemoteConnection::GetConnectSyscallsNumForTest() { - return WriterData.Status.ConnectSyscalls; - } - - void TRemoteConnection::WriterBeforeWriteErrorMessage(TBusMessage* message, EMessageStatus status) { - if (Session->IsSource_) { - CheckedCast<TRemoteClientSession*>(Session.Get())->ReleaseInFlight({message}); - WriterErrorMessage(message, status); - } else { - TBusMessagePtrAndHeader h(message); - CheckedCast<TRemoteServerSession*>(Session.Get())->ReleaseInWorkResponses(MakeArrayRef(&h, 1)); - WriterErrorMessage(h.MessagePtr.Release(), status); - } - } - - void TRemoteConnection::WriterErrorMessage(TNonDestroyingAutoPtr<TBusMessage> m, EMessageStatus status) { - TBusMessage* released = m.Release(); - WriterErrorMessages(MakeArrayRef(&released, 1), status); - } - - void TRemoteConnection::WriterErrorMessages(const TArrayRef<TBusMessage*> ms, EMessageStatus status) { - ResetOneWayFlag(ms); - - WriterData.Status.Incremental.StatusCounter[status] += ms.size(); - for (auto m : ms) { - Session->InvokeOnError(m, status); - } - } - - void TRemoteConnection::FireClientConnectionEvent(TClientConnectionEvent::EType type) { - Y_VERIFY(Session->IsSource_, "state check"); - TClientConnectionEvent event(type, ConnectionId, PeerAddr); - TRemoteClientSession* session = CheckedCast<TRemoteClientSession*>(Session.Get()); - session->ClientHandler->OnClientConnectionEvent(event); - } - - bool TRemoteConnection::IsAlive() const { - return !AtomicGet(WriterData.Down); - } + + if (!WriterData.Buffer.Empty()) { + if (WriterData.Buffer.Size() >= Config.SendThreshold) { + WriterData.State = WRITER_FLUSHING; + } else if (WriterData.CorkUntil == TInstant::Zero()) { + WriterData.State = WRITER_FLUSHING; + } else if (TInstant::Now() >= WriterData.CorkUntil) { + WriterData.State = WRITER_FLUSHING; + } else { + // keep filling + Y_ASSERT(WriterData.State == WRITER_FILLING); + GetWriterSchedulerActor()->ScheduleAt(WriterData.CorkUntil); + } + } else { + // keep filling + Y_ASSERT(WriterData.State == WRITER_FILLING); + } + + size_t bytes = MessageSize(*writeMessages.GetVector()); + + QuotaReturnSelf(writeMessages.GetVector()->size(), bytes); + + // This is called before `send` syscall inducing latency + MessageSent(*writeMessages.GetVector()); + } + + size_t TRemoteConnection::MessageSize(TArrayRef<TBusMessagePtrAndHeader> messages) { + size_t size = 0; + for (const auto& message : messages) + size += message.MessagePtr->RequestSize; + + return size; + } + + size_t TRemoteConnection::GetInFlight() { + return AtomicGet(WriterData.InFlight); + } + + size_t TRemoteConnection::GetConnectSyscallsNumForTest() { + return WriterData.Status.ConnectSyscalls; + } + + void TRemoteConnection::WriterBeforeWriteErrorMessage(TBusMessage* message, EMessageStatus status) { + if (Session->IsSource_) { + CheckedCast<TRemoteClientSession*>(Session.Get())->ReleaseInFlight({message}); + WriterErrorMessage(message, status); + } else { + TBusMessagePtrAndHeader h(message); + CheckedCast<TRemoteServerSession*>(Session.Get())->ReleaseInWorkResponses(MakeArrayRef(&h, 1)); + WriterErrorMessage(h.MessagePtr.Release(), status); + } + } + + void TRemoteConnection::WriterErrorMessage(TNonDestroyingAutoPtr<TBusMessage> m, EMessageStatus status) { + TBusMessage* released = m.Release(); + WriterErrorMessages(MakeArrayRef(&released, 1), status); + } + + void TRemoteConnection::WriterErrorMessages(const TArrayRef<TBusMessage*> ms, EMessageStatus status) { + ResetOneWayFlag(ms); + + WriterData.Status.Incremental.StatusCounter[status] += ms.size(); + for (auto m : ms) { + Session->InvokeOnError(m, status); + } + } + + void TRemoteConnection::FireClientConnectionEvent(TClientConnectionEvent::EType type) { + Y_VERIFY(Session->IsSource_, "state check"); + TClientConnectionEvent event(type, ConnectionId, PeerAddr); + TRemoteClientSession* session = CheckedCast<TRemoteClientSession*>(Session.Get()); + session->ClientHandler->OnClientConnectionEvent(event); + } + + bool TRemoteConnection::IsAlive() const { + return !AtomicGet(WriterData.Down); + } } } diff --git a/library/cpp/messagebus/remote_connection.h b/library/cpp/messagebus/remote_connection.h index 4538947368..575fbff32e 100644 --- a/library/cpp/messagebus/remote_connection.h +++ b/library/cpp/messagebus/remote_connection.h @@ -26,269 +26,269 @@ #include <util/system/event.h> #include <util/thread/lfstack.h> -namespace NBus { - namespace NPrivate { - class TRemoteConnection; - - typedef TIntrusivePtr<TRemoteConnection> TRemoteConnectionPtr; - typedef TIntrusivePtr<TBusSessionImpl> TRemoteSessionPtr; - - static void* const WriteCookie = (void*)1; - static void* const ReadCookie = (void*)2; - - enum { - WAKE_QUOTA_MSG = 0x01, - WAKE_QUOTA_BYTES = 0x02 - }; - - struct TWriterTag {}; - struct TReaderTag {}; - struct TReconnectTag {}; - struct TWakeReaderTag {}; - - struct TWriterToReaderSocketMessage { - TSocket Socket; - ui32 SocketVersion; - - TWriterToReaderSocketMessage(TSocket socket, ui32 socketVersion) - : Socket(socket) - , SocketVersion(socketVersion) - { - } - }; - - class TRemoteConnection - : public NEventLoop::IEventHandler, - public ::NActor::TActor<TRemoteConnection, TWriterTag>, - public ::NActor::TActor<TRemoteConnection, TReaderTag>, - private ::NActor::TQueueInActor<TRemoteConnection, TWriterToReaderSocketMessage, TReaderTag>, - private ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TReconnectTag>, - private ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TWakeReaderTag>, - public TScheduleActor<TRemoteConnection, TWriterTag> { - friend struct TBusSessionImpl; - friend class TRemoteClientSession; - friend class TRemoteServerSession; - friend class ::NActor::TQueueInActor<TRemoteConnection, TWriterToReaderSocketMessage, TReaderTag>; - friend class ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TReconnectTag>; - friend class ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TWakeReaderTag>; - - protected: - ::NActor::TQueueInActor<TRemoteConnection, TWriterToReaderSocketMessage, TReaderTag>* ReaderGetSocketQueue() { - return this; - } - - ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TReconnectTag>* WriterGetReconnectQueue() { - return this; - } - - ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TWakeReaderTag>* WriterGetWakeQueue() { - return this; - } - - protected: - TRemoteConnection(TRemoteSessionPtr session, ui64 connectionId, TNetAddr addr); - ~TRemoteConnection() override; - - virtual void ClearOutgoingQueue(TMessagesPtrs&, bool reconnect /* or shutdown */); - - public: - void Send(TNonDestroyingAutoPtr<TBusMessage> msg); - void Shutdown(EMessageStatus status); - - inline const TNetAddr& GetAddr() const noexcept; - - private: - friend class TScheduleConnect; - friend class TWorkIO; - - protected: - static size_t MessageSize(TArrayRef<TBusMessagePtrAndHeader>); - bool QuotaAcquire(size_t msg, size_t bytes); - void QuotaConsume(size_t msg, size_t bytes); - void QuotaReturnSelf(size_t items, size_t bytes); - bool QuotaReturnValues(size_t items, size_t bytes); - - bool ReaderProcessBuffer(); - bool ReaderFillBuffer(); - void ReaderFlushMessages(); - - void ReadQuotaWakeup(); - ui32 WriteWakeFlags() const; - - virtual bool NeedInterruptRead() { - return false; - } - - public: - virtual void TryConnect(); - void ProcessItem(TReaderTag, ::NActor::TDefaultTag, TWriterToReaderSocketMessage); - void ProcessItem(TWriterTag, TReconnectTag, ui32 socketVersion); - void ProcessItem(TWriterTag, TWakeReaderTag, ui32 awakeFlags); - void Act(TReaderTag); - inline void WriterBeforeWriteErrorMessage(TBusMessage*, EMessageStatus); - void ClearBeforeSendQueue(EMessageStatus reasonForQueues); - void ClearReplyQueue(EMessageStatus reasonForQueues); - inline void ProcessBeforeSendQueueMessage(TBusMessage*, TInstant now); - void ProcessBeforeSendQueue(TInstant now); - void WriterProcessStatusDown(); - void ReaderProcessStatusDown(); - void ProcessWriterDown(); - void DropEnqueuedData(EMessageStatus reason, EMessageStatus reasonForQueues); - const TRemoteConnectionWriterStatus& WriterGetStatus(); - virtual void WriterFillStatus(); - void WriterFillInFlight(); - virtual void BeforeTryWrite(); - void Act(TWriterTag); - void ScheduleRead(); - void ScheduleWrite(); - void ScheduleShutdownOnServerOrReconnectOnClient(EMessageStatus status, bool writer); - void ScheduleShutdown(EMessageStatus status); - void WriterFlushBuffer(); - void WriterFillBuffer(); - void ReaderSendStatus(TInstant now, bool force = false); - const TRemoteConnectionReaderStatus& ReaderFillStatus(); - void WriterRotateCounters(); - void WriterSendStatus(TInstant now, bool force = false); - void WriterSendStatusIfNecessary(TInstant now); - void QuotaReturnAside(size_t items, size_t bytes); - virtual void ReaderProcessMessageUnknownVersion(TArrayRef<const char> dataRef) = 0; - bool MessageRead(TArrayRef<const char> dataRef, TInstant now); - virtual void MessageSent(TArrayRef<TBusMessagePtrAndHeader> messages) = 0; - - void CallSerialize(TBusMessage* msg, TBuffer& buffer) const; - void SerializeMessage(TBusMessage* msg, TBuffer* data, TMessageCounter* counter) const; - TBusMessage* DeserializeMessage(TArrayRef<const char> dataRef, const TBusHeader* header, TMessageCounter* messageCounter, EMessageStatus* status) const; - - void ResetOneWayFlag(TArrayRef<TBusMessage*>); - - inline ::NActor::TActor<TRemoteConnection, TWriterTag>* GetWriterActor() { - return this; - } - inline ::NActor::TActor<TRemoteConnection, TReaderTag>* GetReaderActor() { - return this; - } - inline TScheduleActor<TRemoteConnection, TWriterTag>* GetWriterSchedulerActor() { - return this; - } - - void WriterErrorMessage(TNonDestroyingAutoPtr<TBusMessage> m, EMessageStatus status); - // takes ownership of ms - void WriterErrorMessages(const TArrayRef<TBusMessage*> ms, EMessageStatus status); - - void FireClientConnectionEvent(TClientConnectionEvent::EType); - - size_t GetInFlight(); - size_t GetConnectSyscallsNumForTest(); - - bool IsReturnConnectFailedImmediately() { - return (bool)AtomicGet(ReturnConnectFailedImmediately); - } - - bool IsAlive() const; - - TRemoteSessionPtr Session; - TBusProtocol* const Proto; - TBusSessionConfig const Config; - bool RemovedFromSession; - const ui64 ConnectionId; - const TNetAddr PeerAddr; - const TBusSocketAddr PeerAddrSocketAddr; - - const TInstant CreatedTime; - TInstant LastConnectAttempt; - TAtomic ReturnConnectFailedImmediately; - - protected: - ::NActor::TQueueForActor<TBusMessage*> BeforeSendQueue; - TLockFreeStack<TBusHeader> WrongVersionRequests; - - struct TWriterData { - TAtomic Down; - - NEventLoop::TChannelPtr Channel; - ui32 SocketVersion; - - TRemoteConnectionWriterStatus Status; - TInstant StatusLastSendTime; - - TLocalTasks TimeToRotateCounters; - - TAtomic InFlight; - - TTimedMessages SendQueue; - ui32 AwakeFlags; - EWriterState State; - TLeftRightBuffer Buffer; - TInstant CorkUntil; +namespace NBus { + namespace NPrivate { + class TRemoteConnection; + + typedef TIntrusivePtr<TRemoteConnection> TRemoteConnectionPtr; + typedef TIntrusivePtr<TBusSessionImpl> TRemoteSessionPtr; + + static void* const WriteCookie = (void*)1; + static void* const ReadCookie = (void*)2; + + enum { + WAKE_QUOTA_MSG = 0x01, + WAKE_QUOTA_BYTES = 0x02 + }; + + struct TWriterTag {}; + struct TReaderTag {}; + struct TReconnectTag {}; + struct TWakeReaderTag {}; + + struct TWriterToReaderSocketMessage { + TSocket Socket; + ui32 SocketVersion; + + TWriterToReaderSocketMessage(TSocket socket, ui32 socketVersion) + : Socket(socket) + , SocketVersion(socketVersion) + { + } + }; + + class TRemoteConnection + : public NEventLoop::IEventHandler, + public ::NActor::TActor<TRemoteConnection, TWriterTag>, + public ::NActor::TActor<TRemoteConnection, TReaderTag>, + private ::NActor::TQueueInActor<TRemoteConnection, TWriterToReaderSocketMessage, TReaderTag>, + private ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TReconnectTag>, + private ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TWakeReaderTag>, + public TScheduleActor<TRemoteConnection, TWriterTag> { + friend struct TBusSessionImpl; + friend class TRemoteClientSession; + friend class TRemoteServerSession; + friend class ::NActor::TQueueInActor<TRemoteConnection, TWriterToReaderSocketMessage, TReaderTag>; + friend class ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TReconnectTag>; + friend class ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TWakeReaderTag>; + + protected: + ::NActor::TQueueInActor<TRemoteConnection, TWriterToReaderSocketMessage, TReaderTag>* ReaderGetSocketQueue() { + return this; + } + + ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TReconnectTag>* WriterGetReconnectQueue() { + return this; + } + + ::NActor::TQueueInActor<TRemoteConnection, ui32, TWriterTag, TWakeReaderTag>* WriterGetWakeQueue() { + return this; + } + + protected: + TRemoteConnection(TRemoteSessionPtr session, ui64 connectionId, TNetAddr addr); + ~TRemoteConnection() override; + + virtual void ClearOutgoingQueue(TMessagesPtrs&, bool reconnect /* or shutdown */); + + public: + void Send(TNonDestroyingAutoPtr<TBusMessage> msg); + void Shutdown(EMessageStatus status); + + inline const TNetAddr& GetAddr() const noexcept; + + private: + friend class TScheduleConnect; + friend class TWorkIO; + + protected: + static size_t MessageSize(TArrayRef<TBusMessagePtrAndHeader>); + bool QuotaAcquire(size_t msg, size_t bytes); + void QuotaConsume(size_t msg, size_t bytes); + void QuotaReturnSelf(size_t items, size_t bytes); + bool QuotaReturnValues(size_t items, size_t bytes); + + bool ReaderProcessBuffer(); + bool ReaderFillBuffer(); + void ReaderFlushMessages(); + + void ReadQuotaWakeup(); + ui32 WriteWakeFlags() const; + + virtual bool NeedInterruptRead() { + return false; + } + + public: + virtual void TryConnect(); + void ProcessItem(TReaderTag, ::NActor::TDefaultTag, TWriterToReaderSocketMessage); + void ProcessItem(TWriterTag, TReconnectTag, ui32 socketVersion); + void ProcessItem(TWriterTag, TWakeReaderTag, ui32 awakeFlags); + void Act(TReaderTag); + inline void WriterBeforeWriteErrorMessage(TBusMessage*, EMessageStatus); + void ClearBeforeSendQueue(EMessageStatus reasonForQueues); + void ClearReplyQueue(EMessageStatus reasonForQueues); + inline void ProcessBeforeSendQueueMessage(TBusMessage*, TInstant now); + void ProcessBeforeSendQueue(TInstant now); + void WriterProcessStatusDown(); + void ReaderProcessStatusDown(); + void ProcessWriterDown(); + void DropEnqueuedData(EMessageStatus reason, EMessageStatus reasonForQueues); + const TRemoteConnectionWriterStatus& WriterGetStatus(); + virtual void WriterFillStatus(); + void WriterFillInFlight(); + virtual void BeforeTryWrite(); + void Act(TWriterTag); + void ScheduleRead(); + void ScheduleWrite(); + void ScheduleShutdownOnServerOrReconnectOnClient(EMessageStatus status, bool writer); + void ScheduleShutdown(EMessageStatus status); + void WriterFlushBuffer(); + void WriterFillBuffer(); + void ReaderSendStatus(TInstant now, bool force = false); + const TRemoteConnectionReaderStatus& ReaderFillStatus(); + void WriterRotateCounters(); + void WriterSendStatus(TInstant now, bool force = false); + void WriterSendStatusIfNecessary(TInstant now); + void QuotaReturnAside(size_t items, size_t bytes); + virtual void ReaderProcessMessageUnknownVersion(TArrayRef<const char> dataRef) = 0; + bool MessageRead(TArrayRef<const char> dataRef, TInstant now); + virtual void MessageSent(TArrayRef<TBusMessagePtrAndHeader> messages) = 0; + + void CallSerialize(TBusMessage* msg, TBuffer& buffer) const; + void SerializeMessage(TBusMessage* msg, TBuffer* data, TMessageCounter* counter) const; + TBusMessage* DeserializeMessage(TArrayRef<const char> dataRef, const TBusHeader* header, TMessageCounter* messageCounter, EMessageStatus* status) const; + + void ResetOneWayFlag(TArrayRef<TBusMessage*>); + + inline ::NActor::TActor<TRemoteConnection, TWriterTag>* GetWriterActor() { + return this; + } + inline ::NActor::TActor<TRemoteConnection, TReaderTag>* GetReaderActor() { + return this; + } + inline TScheduleActor<TRemoteConnection, TWriterTag>* GetWriterSchedulerActor() { + return this; + } + + void WriterErrorMessage(TNonDestroyingAutoPtr<TBusMessage> m, EMessageStatus status); + // takes ownership of ms + void WriterErrorMessages(const TArrayRef<TBusMessage*> ms, EMessageStatus status); + + void FireClientConnectionEvent(TClientConnectionEvent::EType); + + size_t GetInFlight(); + size_t GetConnectSyscallsNumForTest(); + + bool IsReturnConnectFailedImmediately() { + return (bool)AtomicGet(ReturnConnectFailedImmediately); + } + + bool IsAlive() const; + + TRemoteSessionPtr Session; + TBusProtocol* const Proto; + TBusSessionConfig const Config; + bool RemovedFromSession; + const ui64 ConnectionId; + const TNetAddr PeerAddr; + const TBusSocketAddr PeerAddrSocketAddr; + + const TInstant CreatedTime; + TInstant LastConnectAttempt; + TAtomic ReturnConnectFailedImmediately; + + protected: + ::NActor::TQueueForActor<TBusMessage*> BeforeSendQueue; + TLockFreeStack<TBusHeader> WrongVersionRequests; + + struct TWriterData { + TAtomic Down; + + NEventLoop::TChannelPtr Channel; + ui32 SocketVersion; + + TRemoteConnectionWriterStatus Status; + TInstant StatusLastSendTime; + + TLocalTasks TimeToRotateCounters; + + TAtomic InFlight; + + TTimedMessages SendQueue; + ui32 AwakeFlags; + EWriterState State; + TLeftRightBuffer Buffer; + TInstant CorkUntil; TSystemEvent ShutdownComplete; - void SetChannel(NEventLoop::TChannelPtr channel); - void DropChannel(); + void SetChannel(NEventLoop::TChannelPtr channel); + void DropChannel(); - TWriterData(); - ~TWriterData(); - }; + TWriterData(); + ~TWriterData(); + }; - struct TReaderData { - TAtomic Down; + struct TReaderData { + TAtomic Down; - NEventLoop::TChannelPtr Channel; - ui32 SocketVersion; + NEventLoop::TChannelPtr Channel; + ui32 SocketVersion; - TRemoteConnectionReaderStatus Status; - TInstant StatusLastSendTime; + TRemoteConnectionReaderStatus Status; + TInstant StatusLastSendTime; - TBuffer Buffer; - size_t Offset; /* offset in read buffer */ - size_t MoreBytes; /* more bytes required from socket */ - TVectorSwaps<TBusMessagePtrAndHeader> ReadMessages; + TBuffer Buffer; + size_t Offset; /* offset in read buffer */ + size_t MoreBytes; /* more bytes required from socket */ + TVectorSwaps<TBusMessagePtrAndHeader> ReadMessages; TSystemEvent ShutdownComplete; - bool BufferMore() const noexcept { - return MoreBytes > 0; - } + bool BufferMore() const noexcept { + return MoreBytes > 0; + } - bool HasBytesInBuf(size_t bytes) noexcept; - void SetChannel(NEventLoop::TChannelPtr channel); - void DropChannel(); + bool HasBytesInBuf(size_t bytes) noexcept; + void SetChannel(NEventLoop::TChannelPtr channel); + void DropChannel(); - TReaderData(); - ~TReaderData(); - }; + TReaderData(); + ~TReaderData(); + }; - // owned by session status actor - struct TGranStatus { - TGranStatus(TDuration gran) - : Writer(gran) - , Reader(gran) - { - } + // owned by session status actor + struct TGranStatus { + TGranStatus(TDuration gran) + : Writer(gran) + , Reader(gran) + { + } - TGranUp<TRemoteConnectionWriterStatus> Writer; - TGranUp<TRemoteConnectionReaderStatus> Reader; - }; + TGranUp<TRemoteConnectionWriterStatus> Writer; + TGranUp<TRemoteConnectionReaderStatus> Reader; + }; - TWriterData WriterData; - TReaderData ReaderData; - TGranStatus GranStatus; - TTokenQuota QuotaMsg; - TTokenQuota QuotaBytes; + TWriterData WriterData; + TReaderData ReaderData; + TGranStatus GranStatus; + TTokenQuota QuotaMsg; + TTokenQuota QuotaBytes; - size_t MaxBufferSize; + size_t MaxBufferSize; - // client connection only - TLockFreeQueueBatch<TBusMessagePtrAndHeader, TVectorSwaps> ReplyQueue; + // client connection only + TLockFreeQueueBatch<TBusMessagePtrAndHeader, TVectorSwaps> ReplyQueue; - EMessageStatus ShutdownReason; + EMessageStatus ShutdownReason; }; - inline const TNetAddr& TRemoteConnection::GetAddr() const noexcept { - return PeerAddr; - } + inline const TNetAddr& TRemoteConnection::GetAddr() const noexcept { + return PeerAddr; + } - typedef TIntrusivePtr<TRemoteConnection> TRemoteConnectionPtr; + typedef TIntrusivePtr<TRemoteConnection> TRemoteConnectionPtr; } -} +} diff --git a/library/cpp/messagebus/remote_connection_status.cpp b/library/cpp/messagebus/remote_connection_status.cpp index 2c48b2a287..05b47e281e 100644 --- a/library/cpp/messagebus/remote_connection_status.cpp +++ b/library/cpp/messagebus/remote_connection_status.cpp @@ -62,7 +62,7 @@ TRemoteConnectionStatusBase::TRemoteConnectionStatusBase() { } -TRemoteConnectionStatusBase& TRemoteConnectionStatusBase ::operator+=(const TRemoteConnectionStatusBase& that) { +TRemoteConnectionStatusBase& TRemoteConnectionStatusBase ::operator+=(const TRemoteConnectionStatusBase& that) { REMOTE_CONNECTION_STATUS_BASE_MAP(STRUCT_FIELD_ADD, ) return *this; } @@ -72,8 +72,8 @@ TRemoteConnectionIncrementalStatusBase::TRemoteConnectionIncrementalStatusBase() { } -TRemoteConnectionIncrementalStatusBase& TRemoteConnectionIncrementalStatusBase::operator+=( - const TRemoteConnectionIncrementalStatusBase& that) { +TRemoteConnectionIncrementalStatusBase& TRemoteConnectionIncrementalStatusBase::operator+=( + const TRemoteConnectionIncrementalStatusBase& that) { REMOTE_CONNECTION_INCREMENTAL_STATUS_BASE_MAP(STRUCT_FIELD_ADD, ) return *this; } @@ -84,8 +84,8 @@ TRemoteConnectionReaderIncrementalStatus::TRemoteConnectionReaderIncrementalStat } TRemoteConnectionReaderIncrementalStatus& TRemoteConnectionReaderIncrementalStatus::operator+=( - const TRemoteConnectionReaderIncrementalStatus& that) { - TRemoteConnectionIncrementalStatusBase::operator+=(that); + const TRemoteConnectionReaderIncrementalStatus& that) { + TRemoteConnectionIncrementalStatusBase::operator+=(that); REMOTE_CONNECTION_READER_INCREMENTAL_STATUS_MAP(STRUCT_FIELD_ADD, ) return *this; } @@ -96,7 +96,7 @@ TRemoteConnectionReaderStatus::TRemoteConnectionReaderStatus() } TRemoteConnectionReaderStatus& TRemoteConnectionReaderStatus::operator+=(const TRemoteConnectionReaderStatus& that) { - TRemoteConnectionStatusBase::operator+=(that); + TRemoteConnectionStatusBase::operator+=(that); REMOTE_CONNECTION_READER_STATUS_MAP(STRUCT_FIELD_ADD, ) return *this; } @@ -107,8 +107,8 @@ TRemoteConnectionWriterIncrementalStatus::TRemoteConnectionWriterIncrementalStat } TRemoteConnectionWriterIncrementalStatus& TRemoteConnectionWriterIncrementalStatus::operator+=( - const TRemoteConnectionWriterIncrementalStatus& that) { - TRemoteConnectionIncrementalStatusBase::operator+=(that); + const TRemoteConnectionWriterIncrementalStatus& that) { + TRemoteConnectionIncrementalStatusBase::operator+=(that); REMOTE_CONNECTION_WRITER_INCREMENTAL_STATUS(STRUCT_FIELD_ADD, ) return *this; } @@ -119,7 +119,7 @@ TRemoteConnectionWriterStatus::TRemoteConnectionWriterStatus() } TRemoteConnectionWriterStatus& TRemoteConnectionWriterStatus::operator+=(const TRemoteConnectionWriterStatus& that) { - TRemoteConnectionStatusBase::operator+=(that); + TRemoteConnectionStatusBase::operator+=(that); REMOTE_CONNECTION_WRITER_STATUS(STRUCT_FIELD_ADD, ) return *this; } @@ -162,9 +162,9 @@ TString TRemoteConnectionStatus::PrintToString() const { if (!Summary) { // TODO: print MyAddr too, but only if it is set ss << WriterStatus.PeerAddr << " (" << WriterStatus.ConnectionId << ")" - << ", writefd=" << WriterStatus.Fd - << ", readfd=" << ReaderStatus.Fd - << Endl; + << ", writefd=" << WriterStatus.Fd + << ", readfd=" << ReaderStatus.Fd + << Endl; if (WriterStatus.Connected) { p.AddRow("connect time", WriterStatus.ConnectTime.ToString()); p.AddRow("writer state", ToCString(WriterStatus.State)); @@ -182,7 +182,7 @@ TString TRemoteConnectionStatus::PrintToString() const { p.AddRow("send queue", LeftPad(WriterStatus.SendQueueSize, 6)); - if (Server) { + if (Server) { p.AddRow("quota msg", LeftPad(ReaderStatus.QuotaMsg, 6)); p.AddRow("quota bytes", LeftPad(ReaderStatus.QuotaBytes, 6)); p.AddRow("quota exhausted", LeftPad(ReaderStatus.QuotaExhausted, 6)); @@ -254,7 +254,7 @@ TString TSessionDumpStatus::PrintToString() const { return ss.Str(); } -TString TBusMessageQueueStatus::PrintToString() const { +TString TBusMessageQueueStatus::PrintToString() const { TStringStream ss; ss << "work queue:\n"; ss << ExecutorStatus.Status; diff --git a/library/cpp/messagebus/remote_connection_status.h b/library/cpp/messagebus/remote_connection_status.h index 5db10e51ea..f98f1ff8e9 100644 --- a/library/cpp/messagebus/remote_connection_status.h +++ b/library/cpp/messagebus/remote_connection_status.h @@ -15,200 +15,200 @@ namespace NBus { class TConnectionStatusMonRecord; } -namespace NBus { - namespace NPrivate { +namespace NBus { + namespace NPrivate { #define WRITER_STATE_MAP(XX) \ - XX(WRITER_UNKNOWN) \ - XX(WRITER_FILLING) \ - XX(WRITER_FLUSHING) \ + XX(WRITER_UNKNOWN) \ + XX(WRITER_FILLING) \ + XX(WRITER_FLUSHING) \ /**/ - // TODO: move elsewhere - enum EWriterState { - WRITER_STATE_MAP(ENUM_VALUE_GEN_NO_VALUE) - }; + // TODO: move elsewhere + enum EWriterState { + WRITER_STATE_MAP(ENUM_VALUE_GEN_NO_VALUE) + }; - ENUM_TO_STRING(EWriterState, WRITER_STATE_MAP) + ENUM_TO_STRING(EWriterState, WRITER_STATE_MAP) #define STRUCT_FIELD_ADD(name, type, func) func(name, that.name); - template <typename T> - void Reset(T& t) { - t.~T(); - new (&t) T(); - } + template <typename T> + void Reset(T& t) { + t.~T(); + new (&t) T(); + } -#define DURATION_COUNTER_MAP(XX, comma) \ - XX(Count, unsigned, Add) \ - comma \ - XX(SumDuration, TDuration, Add) comma \ - XX(MaxDuration, TDuration, Max) /**/ +#define DURATION_COUNTER_MAP(XX, comma) \ + XX(Count, unsigned, Add) \ + comma \ + XX(SumDuration, TDuration, Add) comma \ + XX(MaxDuration, TDuration, Max) /**/ - struct TDurationCounter { - DURATION_COUNTER_MAP(STRUCT_FIELD_GEN, ) + struct TDurationCounter { + DURATION_COUNTER_MAP(STRUCT_FIELD_GEN, ) - TDuration AvgDuration() const; + TDuration AvgDuration() const; - TDurationCounter(); + TDurationCounter(); - void AddDuration(TDuration d) { - Count += 1; - SumDuration += d; - if (d > MaxDuration) { - MaxDuration = d; - } - } + void AddDuration(TDuration d) { + Count += 1; + SumDuration += d; + if (d > MaxDuration) { + MaxDuration = d; + } + } - TDurationCounter& operator+=(const TDurationCounter&); + TDurationCounter& operator+=(const TDurationCounter&); - TString ToString() const; - }; + TString ToString() const; + }; #define REMOTE_CONNECTION_STATUS_BASE_MAP(XX, comma) \ - XX(ConnectionId, ui64, AssertZero) \ - comma \ - XX(Fd, SOCKET, AssertZero) comma \ - XX(Acts, ui64, Add) comma \ - XX(BufferSize, ui64, Add) /**/ + XX(ConnectionId, ui64, AssertZero) \ + comma \ + XX(Fd, SOCKET, AssertZero) comma \ + XX(Acts, ui64, Add) comma \ + XX(BufferSize, ui64, Add) /**/ - struct TRemoteConnectionStatusBase { - REMOTE_CONNECTION_STATUS_BASE_MAP(STRUCT_FIELD_GEN, ) + struct TRemoteConnectionStatusBase { + REMOTE_CONNECTION_STATUS_BASE_MAP(STRUCT_FIELD_GEN, ) - TRemoteConnectionStatusBase& operator+=(const TRemoteConnectionStatusBase&); + TRemoteConnectionStatusBase& operator+=(const TRemoteConnectionStatusBase&); - TRemoteConnectionStatusBase(); - }; + TRemoteConnectionStatusBase(); + }; #define REMOTE_CONNECTION_INCREMENTAL_STATUS_BASE_MAP(XX, comma) \ - XX(BufferDrops, unsigned, Add) \ - comma \ - XX(NetworkOps, unsigned, Add) /**/ + XX(BufferDrops, unsigned, Add) \ + comma \ + XX(NetworkOps, unsigned, Add) /**/ - struct TRemoteConnectionIncrementalStatusBase { - REMOTE_CONNECTION_INCREMENTAL_STATUS_BASE_MAP(STRUCT_FIELD_GEN, ) + struct TRemoteConnectionIncrementalStatusBase { + REMOTE_CONNECTION_INCREMENTAL_STATUS_BASE_MAP(STRUCT_FIELD_GEN, ) - TRemoteConnectionIncrementalStatusBase& operator+=(const TRemoteConnectionIncrementalStatusBase&); + TRemoteConnectionIncrementalStatusBase& operator+=(const TRemoteConnectionIncrementalStatusBase&); - TRemoteConnectionIncrementalStatusBase(); - }; + TRemoteConnectionIncrementalStatusBase(); + }; #define REMOTE_CONNECTION_READER_INCREMENTAL_STATUS_MAP(XX, comma) \ - XX(MessageCounter, TMessageCounter, Add) \ - comma \ - XX(StatusCounter, TMessageStatusCounter, Add) /**/ + XX(MessageCounter, TMessageCounter, Add) \ + comma \ + XX(StatusCounter, TMessageStatusCounter, Add) /**/ - struct TRemoteConnectionReaderIncrementalStatus: public TRemoteConnectionIncrementalStatusBase { - REMOTE_CONNECTION_READER_INCREMENTAL_STATUS_MAP(STRUCT_FIELD_GEN, ) + struct TRemoteConnectionReaderIncrementalStatus: public TRemoteConnectionIncrementalStatusBase { + REMOTE_CONNECTION_READER_INCREMENTAL_STATUS_MAP(STRUCT_FIELD_GEN, ) - TRemoteConnectionReaderIncrementalStatus& operator+=(const TRemoteConnectionReaderIncrementalStatus&); + TRemoteConnectionReaderIncrementalStatus& operator+=(const TRemoteConnectionReaderIncrementalStatus&); - TRemoteConnectionReaderIncrementalStatus(); - }; + TRemoteConnectionReaderIncrementalStatus(); + }; #define REMOTE_CONNECTION_READER_STATUS_MAP(XX, comma) \ - XX(QuotaMsg, size_t, Add) \ - comma \ - XX(QuotaBytes, size_t, Add) comma \ - XX(QuotaExhausted, size_t, Add) comma \ - XX(Incremental, TRemoteConnectionReaderIncrementalStatus, Add) /**/ + XX(QuotaMsg, size_t, Add) \ + comma \ + XX(QuotaBytes, size_t, Add) comma \ + XX(QuotaExhausted, size_t, Add) comma \ + XX(Incremental, TRemoteConnectionReaderIncrementalStatus, Add) /**/ - struct TRemoteConnectionReaderStatus: public TRemoteConnectionStatusBase { - REMOTE_CONNECTION_READER_STATUS_MAP(STRUCT_FIELD_GEN, ) + struct TRemoteConnectionReaderStatus: public TRemoteConnectionStatusBase { + REMOTE_CONNECTION_READER_STATUS_MAP(STRUCT_FIELD_GEN, ) - TRemoteConnectionReaderStatus& operator+=(const TRemoteConnectionReaderStatus&); + TRemoteConnectionReaderStatus& operator+=(const TRemoteConnectionReaderStatus&); - TRemoteConnectionReaderStatus(); - }; + TRemoteConnectionReaderStatus(); + }; #define REMOTE_CONNECTION_WRITER_INCREMENTAL_STATUS(XX, comma) \ - XX(MessageCounter, TMessageCounter, Add) \ - comma \ - XX(StatusCounter, TMessageStatusCounter, Add) comma \ - XX(ProcessDurationHistogram, TDurationHistogram, Add) /**/ + XX(MessageCounter, TMessageCounter, Add) \ + comma \ + XX(StatusCounter, TMessageStatusCounter, Add) comma \ + XX(ProcessDurationHistogram, TDurationHistogram, Add) /**/ - struct TRemoteConnectionWriterIncrementalStatus: public TRemoteConnectionIncrementalStatusBase { - REMOTE_CONNECTION_WRITER_INCREMENTAL_STATUS(STRUCT_FIELD_GEN, ) + struct TRemoteConnectionWriterIncrementalStatus: public TRemoteConnectionIncrementalStatusBase { + REMOTE_CONNECTION_WRITER_INCREMENTAL_STATUS(STRUCT_FIELD_GEN, ) - TRemoteConnectionWriterIncrementalStatus& operator+=(const TRemoteConnectionWriterIncrementalStatus&); + TRemoteConnectionWriterIncrementalStatus& operator+=(const TRemoteConnectionWriterIncrementalStatus&); - TRemoteConnectionWriterIncrementalStatus(); - }; + TRemoteConnectionWriterIncrementalStatus(); + }; -#define REMOTE_CONNECTION_WRITER_STATUS(XX, comma) \ - XX(Connected, bool, AssertZero) \ - comma \ - XX(ConnectTime, TInstant, AssertZero) comma /* either connect time on client or accept time on server */ \ - XX(ConnectError, int, AssertZero) comma \ - XX(ConnectSyscalls, unsigned, Add) comma \ - XX(PeerAddr, TNetAddr, AssertZero) comma \ - XX(MyAddr, TNetAddr, AssertZero) comma \ - XX(State, EWriterState, AssertZero) comma \ - XX(SendQueueSize, size_t, Add) comma \ - XX(AckMessagesSize, size_t, Add) comma /* client only */ \ - XX(DurationCounter, TDurationCounter, Add) comma /* server only */ \ - XX(DurationCounterPrev, TDurationCounter, Add) comma /* server only */ \ - XX(Incremental, TRemoteConnectionWriterIncrementalStatus, Add) comma \ - XX(ReaderWakeups, size_t, Add) /**/ +#define REMOTE_CONNECTION_WRITER_STATUS(XX, comma) \ + XX(Connected, bool, AssertZero) \ + comma \ + XX(ConnectTime, TInstant, AssertZero) comma /* either connect time on client or accept time on server */ \ + XX(ConnectError, int, AssertZero) comma \ + XX(ConnectSyscalls, unsigned, Add) comma \ + XX(PeerAddr, TNetAddr, AssertZero) comma \ + XX(MyAddr, TNetAddr, AssertZero) comma \ + XX(State, EWriterState, AssertZero) comma \ + XX(SendQueueSize, size_t, Add) comma \ + XX(AckMessagesSize, size_t, Add) comma /* client only */ \ + XX(DurationCounter, TDurationCounter, Add) comma /* server only */ \ + XX(DurationCounterPrev, TDurationCounter, Add) comma /* server only */ \ + XX(Incremental, TRemoteConnectionWriterIncrementalStatus, Add) comma \ + XX(ReaderWakeups, size_t, Add) /**/ - struct TRemoteConnectionWriterStatus: public TRemoteConnectionStatusBase { - REMOTE_CONNECTION_WRITER_STATUS(STRUCT_FIELD_GEN, ) + struct TRemoteConnectionWriterStatus: public TRemoteConnectionStatusBase { + REMOTE_CONNECTION_WRITER_STATUS(STRUCT_FIELD_GEN, ) - TRemoteConnectionWriterStatus(); + TRemoteConnectionWriterStatus(); - TRemoteConnectionWriterStatus& operator+=(const TRemoteConnectionWriterStatus&); + TRemoteConnectionWriterStatus& operator+=(const TRemoteConnectionWriterStatus&); - size_t GetInFlight() const; - }; + size_t GetInFlight() const; + }; #define REMOTE_CONNECTION_STATUS_MAP(XX, comma) \ - XX(Summary, bool) \ - comma \ - XX(Server, bool) /**/ - - struct TRemoteConnectionStatus { - REMOTE_CONNECTION_STATUS_MAP(STRUCT_FIELD_GEN, ) - - TRemoteConnectionReaderStatus ReaderStatus; - TRemoteConnectionWriterStatus WriterStatus; - - TRemoteConnectionStatus(); - - TString PrintToString() const; - TConnectionStatusMonRecord GetStatusProtobuf() const; - }; - - struct TBusSessionStatus { - size_t InFlightCount; - size_t InFlightSize; - bool InputPaused; - - TBusSessionStatus(); - }; - - struct TSessionDumpStatus { - bool Shutdown; - TString Head; - TString Acceptors; - TString ConnectionsSummary; - TString Connections; - TBusSessionStatus Status; - TRemoteConnectionStatus ConnectionStatusSummary; - TBusSessionConfig Config; - - TSessionDumpStatus() - : Shutdown(false) - { - } - - TString PrintToString() const; - }; - - // without sessions - struct TBusMessageQueueStatus { - NActor::NPrivate::TExecutorStatus ExecutorStatus; - TBusQueueConfig Config; - - TString PrintToString() const; - }; - } -} + XX(Summary, bool) \ + comma \ + XX(Server, bool) /**/ + + struct TRemoteConnectionStatus { + REMOTE_CONNECTION_STATUS_MAP(STRUCT_FIELD_GEN, ) + + TRemoteConnectionReaderStatus ReaderStatus; + TRemoteConnectionWriterStatus WriterStatus; + + TRemoteConnectionStatus(); + + TString PrintToString() const; + TConnectionStatusMonRecord GetStatusProtobuf() const; + }; + + struct TBusSessionStatus { + size_t InFlightCount; + size_t InFlightSize; + bool InputPaused; + + TBusSessionStatus(); + }; + + struct TSessionDumpStatus { + bool Shutdown; + TString Head; + TString Acceptors; + TString ConnectionsSummary; + TString Connections; + TBusSessionStatus Status; + TRemoteConnectionStatus ConnectionStatusSummary; + TBusSessionConfig Config; + + TSessionDumpStatus() + : Shutdown(false) + { + } + + TString PrintToString() const; + }; + + // without sessions + struct TBusMessageQueueStatus { + NActor::NPrivate::TExecutorStatus ExecutorStatus; + TBusQueueConfig Config; + + TString PrintToString() const; + }; + } +} diff --git a/library/cpp/messagebus/remote_server_connection.cpp b/library/cpp/messagebus/remote_server_connection.cpp index 74be34ded9..17df4dd4a3 100644 --- a/library/cpp/messagebus/remote_server_connection.cpp +++ b/library/cpp/messagebus/remote_server_connection.cpp @@ -15,7 +15,7 @@ TRemoteServerConnection::TRemoteServerConnection(TRemoteServerSessionPtr session { } -void TRemoteServerConnection::Init(SOCKET socket, TInstant now) { +void TRemoteServerConnection::Init(SOCKET socket, TInstant now) { WriterData.Status.ConnectTime = now; WriterData.Status.Connected = true; diff --git a/library/cpp/messagebus/remote_server_connection.h b/library/cpp/messagebus/remote_server_connection.h index 63d7f20646..925338f3ff 100644 --- a/library/cpp/messagebus/remote_server_connection.h +++ b/library/cpp/messagebus/remote_server_connection.h @@ -4,29 +4,29 @@ #include <util/generic/object_counter.h> -namespace NBus { - namespace NPrivate { - class TRemoteServerConnection: public TRemoteConnection { - friend struct TBusSessionImpl; - friend class TRemoteServerSession; +namespace NBus { + namespace NPrivate { + class TRemoteServerConnection: public TRemoteConnection { + friend struct TBusSessionImpl; + friend class TRemoteServerSession; - TObjectCounter<TRemoteServerConnection> ObjectCounter; + TObjectCounter<TRemoteServerConnection> ObjectCounter; - public: - TRemoteServerConnection(TRemoteServerSessionPtr session, ui64 id, TNetAddr addr); + public: + TRemoteServerConnection(TRemoteServerSessionPtr session, ui64 id, TNetAddr addr); - void Init(SOCKET socket, TInstant now); + void Init(SOCKET socket, TInstant now); - inline TRemoteServerSession* GetSession(); + inline TRemoteServerSession* GetSession(); - void HandleEvent(SOCKET socket, void* cookie) override; + void HandleEvent(SOCKET socket, void* cookie) override; - bool NeedInterruptRead() override; + bool NeedInterruptRead() override; - void MessageSent(TArrayRef<TBusMessagePtrAndHeader> messages) override; + void MessageSent(TArrayRef<TBusMessagePtrAndHeader> messages) override; - void ReaderProcessMessageUnknownVersion(TArrayRef<const char> dataRef) override; - }; + void ReaderProcessMessageUnknownVersion(TArrayRef<const char> dataRef) override; + }; - } -} + } +} diff --git a/library/cpp/messagebus/remote_server_session.cpp b/library/cpp/messagebus/remote_server_session.cpp index 6abbf88a60..0bc2853e94 100644 --- a/library/cpp/messagebus/remote_server_session.cpp +++ b/library/cpp/messagebus/remote_server_session.cpp @@ -16,47 +16,47 @@ using namespace NBus; using namespace NBus::NPrivate; TRemoteServerSession::TRemoteServerSession(TBusMessageQueue* queue, - TBusProtocol* proto, IBusServerHandler* handler, - const TBusServerSessionConfig& config, const TString& name) + TBusProtocol* proto, IBusServerHandler* handler, + const TBusServerSessionConfig& config, const TString& name) : TBusSessionImpl(false, queue, proto, handler, config, name) , ServerOwnedMessages(config.MaxInFlight, config.MaxInFlightBySize, "ServerOwnedMessages") , ServerHandler(handler) { - if (config.PerConnectionMaxInFlightBySize > 0) { - if (config.PerConnectionMaxInFlightBySize < config.MaxMessageSize) + if (config.PerConnectionMaxInFlightBySize > 0) { + if (config.PerConnectionMaxInFlightBySize < config.MaxMessageSize) ythrow yexception() << "too low PerConnectionMaxInFlightBySize value"; } } -namespace NBus { - namespace NPrivate { - class TInvokeOnMessage: public IWorkItem { - private: - TRemoteServerSession* RemoteServerSession; - TBusMessagePtrAndHeader Request; - TIntrusivePtr<TRemoteServerConnection> Connection; - - public: - TInvokeOnMessage(TRemoteServerSession* session, TBusMessagePtrAndHeader& request, TIntrusivePtr<TRemoteServerConnection>& connection) - : RemoteServerSession(session) - { - Y_ASSERT(!!connection); - Connection.Swap(connection); - - Request.Swap(request); - } - - void DoWork() override { - THolder<TInvokeOnMessage> holder(this); - RemoteServerSession->InvokeOnMessage(Request, Connection); - // TODO: TRemoteServerSessionSemaphore should be enough - RemoteServerSession->JobCount.Decrement(); - } - }; +namespace NBus { + namespace NPrivate { + class TInvokeOnMessage: public IWorkItem { + private: + TRemoteServerSession* RemoteServerSession; + TBusMessagePtrAndHeader Request; + TIntrusivePtr<TRemoteServerConnection> Connection; - } -} + public: + TInvokeOnMessage(TRemoteServerSession* session, TBusMessagePtrAndHeader& request, TIntrusivePtr<TRemoteServerConnection>& connection) + : RemoteServerSession(session) + { + Y_ASSERT(!!connection); + Connection.Swap(connection); + + Request.Swap(request); + } + + void DoWork() override { + THolder<TInvokeOnMessage> holder(this); + RemoteServerSession->InvokeOnMessage(Request, Connection); + // TODO: TRemoteServerSessionSemaphore should be enough + RemoteServerSession->JobCount.Decrement(); + } + }; + + } +} void TRemoteServerSession::OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& messages) { AcquireInWorkRequests(messages); @@ -165,7 +165,7 @@ void TRemoteServerSession::ReleaseInWorkResponses(TArrayRef<const TBusMessagePtr ServerOwnedMessages.ReleaseMultiple(responses.size(), size); } -void TRemoteServerSession::ReleaseInWorkRequests(TRemoteConnection& con, TBusMessage* request) { +void TRemoteServerSession::ReleaseInWorkRequests(TRemoteConnection& con, TBusMessage* request) { Y_ASSERT((request->LocalFlags & MESSAGE_IN_WORK)); request->LocalFlags &= ~MESSAGE_IN_WORK; @@ -175,7 +175,7 @@ void TRemoteServerSession::ReleaseInWorkRequests(TRemoteConnection& con, TBusMes ServerOwnedMessages.ReleaseMultiple(1, size); } -void TRemoteServerSession::ReleaseInWork(TBusIdentity& ident) { +void TRemoteServerSession::ReleaseInWork(TBusIdentity& ident) { ident.SetInWork(false); ident.Connection->QuotaReturnAside(1, ident.Size); diff --git a/library/cpp/messagebus/remote_server_session.h b/library/cpp/messagebus/remote_server_session.h index f5c266a7f7..5de07c4e59 100644 --- a/library/cpp/messagebus/remote_server_session.h +++ b/library/cpp/messagebus/remote_server_session.h @@ -5,50 +5,50 @@ #ifdef _MSC_VER #pragma warning(push) -#pragma warning(disable : 4250) // 'NBus::NPrivate::TRemoteClientSession' : inherits 'NBus::NPrivate::TBusSessionImpl::NBus::NPrivate::TBusSessionImpl::GetConfig' via dominance +#pragma warning(disable : 4250) // 'NBus::NPrivate::TRemoteClientSession' : inherits 'NBus::NPrivate::TBusSessionImpl::NBus::NPrivate::TBusSessionImpl::GetConfig' via dominance #endif -namespace NBus { - namespace NPrivate { - class TRemoteServerSession: public TBusServerSession, public TBusSessionImpl { - friend class TRemoteServerConnection; +namespace NBus { + namespace NPrivate { + class TRemoteServerSession: public TBusServerSession, public TBusSessionImpl { + friend class TRemoteServerConnection; - private: - TObjectCounter<TRemoteServerSession> ObjectCounter; + private: + TObjectCounter<TRemoteServerSession> ObjectCounter; - TRemoteServerSessionSemaphore ServerOwnedMessages; - IBusServerHandler* const ServerHandler; + TRemoteServerSessionSemaphore ServerOwnedMessages; + IBusServerHandler* const ServerHandler; - public: - TRemoteServerSession(TBusMessageQueue* queue, TBusProtocol* proto, - IBusServerHandler* handler, - const TBusSessionConfig& config, const TString& name); + public: + TRemoteServerSession(TBusMessageQueue* queue, TBusProtocol* proto, + IBusServerHandler* handler, + const TBusSessionConfig& config, const TString& name); - void OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& newMsg) override; - void InvokeOnMessage(TBusMessagePtrAndHeader& request, TIntrusivePtr<TRemoteServerConnection>& conn); + void OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& newMsg) override; + void InvokeOnMessage(TBusMessagePtrAndHeader& request, TIntrusivePtr<TRemoteServerConnection>& conn); - EMessageStatus SendReply(const TBusIdentity& ident, TBusMessage* pRep) override; + EMessageStatus SendReply(const TBusIdentity& ident, TBusMessage* pRep) override; - EMessageStatus ForgetRequest(const TBusIdentity& ident) override; + EMessageStatus ForgetRequest(const TBusIdentity& ident) override; - int GetInFlight() const noexcept override; - void FillStatus() override; + int GetInFlight() const noexcept override; + void FillStatus() override; - void Shutdown() override; + void Shutdown() override; - void PauseInput(bool pause) override; - unsigned GetActualListenPort() override; + void PauseInput(bool pause) override; + unsigned GetActualListenPort() override; - void AcquireInWorkRequests(TArrayRef<const TBusMessagePtrAndHeader> requests); - void ReleaseInWorkResponses(TArrayRef<const TBusMessagePtrAndHeader> responses); - void ReleaseInWorkRequests(TRemoteConnection&, TBusMessage*); - void ReleaseInWork(TBusIdentity&); - void ConvertInWork(TBusIdentity& req, TBusMessage* reply); - }; + void AcquireInWorkRequests(TArrayRef<const TBusMessagePtrAndHeader> requests); + void ReleaseInWorkResponses(TArrayRef<const TBusMessagePtrAndHeader> responses); + void ReleaseInWorkRequests(TRemoteConnection&, TBusMessage*); + void ReleaseInWork(TBusIdentity&); + void ConvertInWork(TBusIdentity& req, TBusMessage* reply); + }; #ifdef _MSC_VER #pragma warning(pop) #endif - } -} + } +} diff --git a/library/cpp/messagebus/remote_server_session_semaphore.cpp b/library/cpp/messagebus/remote_server_session_semaphore.cpp index 6094a3586e..3287c869c8 100644 --- a/library/cpp/messagebus/remote_server_session_semaphore.cpp +++ b/library/cpp/messagebus/remote_server_session_semaphore.cpp @@ -7,7 +7,7 @@ using namespace NBus; using namespace NBus::NPrivate; TRemoteServerSessionSemaphore::TRemoteServerSessionSemaphore( - TAtomicBase limitCount, TAtomicBase limitSize, const char* name) + TAtomicBase limitCount, TAtomicBase limitSize, const char* name) : Name(name) , LimitCount(limitCount) , LimitSize(limitSize) @@ -20,7 +20,7 @@ TRemoteServerSessionSemaphore::TRemoteServerSessionSemaphore( Y_UNUSED(Name); } -TRemoteServerSessionSemaphore::~TRemoteServerSessionSemaphore() { +TRemoteServerSessionSemaphore::~TRemoteServerSessionSemaphore() { Y_VERIFY(AtomicGet(CurrentCount) == 0); // TODO: fix spider and enable //Y_VERIFY(AtomicGet(CurrentSize) == 0); diff --git a/library/cpp/messagebus/remote_server_session_semaphore.h b/library/cpp/messagebus/remote_server_session_semaphore.h index de714fd342..32b0f27acc 100644 --- a/library/cpp/messagebus/remote_server_session_semaphore.h +++ b/library/cpp/messagebus/remote_server_session_semaphore.h @@ -4,39 +4,39 @@ #include <util/generic/noncopyable.h> -namespace NBus { - namespace NPrivate { - class TRemoteServerSessionSemaphore: public TComplexConditionSemaphore<TRemoteServerSessionSemaphore> { - private: - const char* const Name; - - TAtomicBase const LimitCount; - TAtomicBase const LimitSize; - TAtomic CurrentCount; - TAtomic CurrentSize; - TAtomic PausedByUser; - TAtomic StopSignal; - - public: - TRemoteServerSessionSemaphore(TAtomicBase limitCount, TAtomicBase limitSize, const char* name = "unnamed"); - ~TRemoteServerSessionSemaphore(); - - TAtomicBase GetCurrentCount() const { - return AtomicGet(CurrentCount); - } - TAtomicBase GetCurrentSize() const { - return AtomicGet(CurrentSize); - } - - void IncrementMultiple(TAtomicBase count, TAtomicBase size); - bool TryWait(); - void ReleaseMultiple(TAtomicBase count, TAtomicBase size); - void Stop(); - void PauseByUsed(bool pause); - - private: - void CheckNeedToUnlock(); - }; - - } -} +namespace NBus { + namespace NPrivate { + class TRemoteServerSessionSemaphore: public TComplexConditionSemaphore<TRemoteServerSessionSemaphore> { + private: + const char* const Name; + + TAtomicBase const LimitCount; + TAtomicBase const LimitSize; + TAtomic CurrentCount; + TAtomic CurrentSize; + TAtomic PausedByUser; + TAtomic StopSignal; + + public: + TRemoteServerSessionSemaphore(TAtomicBase limitCount, TAtomicBase limitSize, const char* name = "unnamed"); + ~TRemoteServerSessionSemaphore(); + + TAtomicBase GetCurrentCount() const { + return AtomicGet(CurrentCount); + } + TAtomicBase GetCurrentSize() const { + return AtomicGet(CurrentSize); + } + + void IncrementMultiple(TAtomicBase count, TAtomicBase size); + bool TryWait(); + void ReleaseMultiple(TAtomicBase count, TAtomicBase size); + void Stop(); + void PauseByUsed(bool pause); + + private: + void CheckNeedToUnlock(); + }; + + } +} diff --git a/library/cpp/messagebus/scheduler/scheduler.cpp b/library/cpp/messagebus/scheduler/scheduler.cpp index 5a5fe52894..f4122e6f9f 100644 --- a/library/cpp/messagebus/scheduler/scheduler.cpp +++ b/library/cpp/messagebus/scheduler/scheduler.cpp @@ -11,14 +11,14 @@ using namespace NBus::NPrivate; class TScheduleDeadlineCompare { public: - bool operator()(const IScheduleItemAutoPtr& i1, const IScheduleItemAutoPtr& i2) const noexcept { + bool operator()(const IScheduleItemAutoPtr& i1, const IScheduleItemAutoPtr& i2) const noexcept { return i1->GetScheduleTime() > i2->GetScheduleTime(); } }; TScheduler::TScheduler() : StopThread(false) - , Thread([&] { this->SchedulerThread(); }) + , Thread([&] { this->SchedulerThread(); }) { } diff --git a/library/cpp/messagebus/scheduler/scheduler.h b/library/cpp/messagebus/scheduler/scheduler.h index afcc0de55d..0a2d1228c2 100644 --- a/library/cpp/messagebus/scheduler/scheduler.h +++ b/library/cpp/messagebus/scheduler/scheduler.h @@ -11,58 +11,58 @@ #include <util/system/mutex.h> #include <util/system/thread.h> -namespace NBus { - namespace NPrivate { - class IScheduleItem { - public: - inline IScheduleItem(TInstant scheduleTime) noexcept; - virtual ~IScheduleItem() { - } +namespace NBus { + namespace NPrivate { + class IScheduleItem { + public: + inline IScheduleItem(TInstant scheduleTime) noexcept; + virtual ~IScheduleItem() { + } - virtual void Do() = 0; - inline TInstant GetScheduleTime() const noexcept; + virtual void Do() = 0; + inline TInstant GetScheduleTime() const noexcept; - private: - TInstant ScheduleTime; - }; + private: + TInstant ScheduleTime; + }; - using IScheduleItemAutoPtr = TAutoPtr<IScheduleItem>; + using IScheduleItemAutoPtr = TAutoPtr<IScheduleItem>; - class TScheduler { - public: - TScheduler(); - ~TScheduler(); - void Stop(); - void Schedule(TAutoPtr<IScheduleItem> i); + class TScheduler { + public: + TScheduler(); + ~TScheduler(); + void Stop(); + void Schedule(TAutoPtr<IScheduleItem> i); - size_t Size() const; + size_t Size() const; - private: - void SchedulerThread(); + private: + void SchedulerThread(); - void FillNextItem(); + void FillNextItem(); - private: - TVector<IScheduleItemAutoPtr> Items; - IScheduleItemAutoPtr NextItem; - typedef TMutex TLock; - TLock Lock; - TCondVar CondVar; + private: + TVector<IScheduleItemAutoPtr> Items; + IScheduleItemAutoPtr NextItem; + typedef TMutex TLock; + TLock Lock; + TCondVar CondVar; - TObjectCounter<TScheduler> ObjectCounter; + TObjectCounter<TScheduler> ObjectCounter; - bool StopThread; - NThreading::TLegacyFuture<> Thread; - }; + bool StopThread; + NThreading::TLegacyFuture<> Thread; + }; - inline IScheduleItem::IScheduleItem(TInstant scheduleTime) noexcept - : ScheduleTime(scheduleTime) - { - } + inline IScheduleItem::IScheduleItem(TInstant scheduleTime) noexcept + : ScheduleTime(scheduleTime) + { + } - inline TInstant IScheduleItem::GetScheduleTime() const noexcept { - return ScheduleTime; - } + inline TInstant IScheduleItem::GetScheduleTime() const noexcept { + return ScheduleTime; + } - } + } } diff --git a/library/cpp/messagebus/scheduler/scheduler_ut.cpp b/library/cpp/messagebus/scheduler/scheduler_ut.cpp index a5ea641c10..f6d69ceffb 100644 --- a/library/cpp/messagebus/scheduler/scheduler_ut.cpp +++ b/library/cpp/messagebus/scheduler/scheduler_ut.cpp @@ -14,8 +14,8 @@ Y_UNIT_TEST_SUITE(TSchedulerTests) { TSimpleScheduleItem(TTestSync* testSync) : IScheduleItem((TInstant::Now() + TDuration::MilliSeconds(1))) , TestSync(testSync) - { - } + { + } void Do() override { TestSync->WaitForAndIncrement(0); diff --git a/library/cpp/messagebus/scheduler_actor.h b/library/cpp/messagebus/scheduler_actor.h index d0c23c94c4..4176ea45a9 100644 --- a/library/cpp/messagebus/scheduler_actor.h +++ b/library/cpp/messagebus/scheduler_actor.h @@ -8,78 +8,78 @@ #include <util/system/mutex.h> -namespace NBus { - namespace NPrivate { - template <typename TThis, typename TTag = NActor::TDefaultTag> - class TScheduleActor { - typedef NActor::TActor<TThis, TTag> TActorForMe; - - private: - TScheduler* const Scheduler; - - TMutex Mutex; - - TInstant ScheduleTime; - - public: - TLocalTasks Alarm; - - private: - struct TScheduleItemImpl: public IScheduleItem { - TIntrusivePtr<TThis> Thiz; - - TScheduleItemImpl(TIntrusivePtr<TThis> thiz, TInstant when) - : IScheduleItem(when) - , Thiz(thiz) - { - } - - void Do() override { - { - TWhatThreadDoesAcquireGuard<TMutex> guard(Thiz->Mutex, "scheduler actor: acquiring lock for Do"); - - if (Thiz->ScheduleTime == TInstant::Max()) { - // was already fired - return; - } - - Thiz->ScheduleTime = TInstant::Max(); - } - - Thiz->Alarm.AddTask(); - Thiz->GetActorForMe()->Schedule(); - } - }; - - public: - TScheduleActor(TScheduler* scheduler) - : Scheduler(scheduler) - , ScheduleTime(TInstant::Max()) +namespace NBus { + namespace NPrivate { + template <typename TThis, typename TTag = NActor::TDefaultTag> + class TScheduleActor { + typedef NActor::TActor<TThis, TTag> TActorForMe; + + private: + TScheduler* const Scheduler; + + TMutex Mutex; + + TInstant ScheduleTime; + + public: + TLocalTasks Alarm; + + private: + struct TScheduleItemImpl: public IScheduleItem { + TIntrusivePtr<TThis> Thiz; + + TScheduleItemImpl(TIntrusivePtr<TThis> thiz, TInstant when) + : IScheduleItem(when) + , Thiz(thiz) + { + } + + void Do() override { + { + TWhatThreadDoesAcquireGuard<TMutex> guard(Thiz->Mutex, "scheduler actor: acquiring lock for Do"); + + if (Thiz->ScheduleTime == TInstant::Max()) { + // was already fired + return; + } + + Thiz->ScheduleTime = TInstant::Max(); + } + + Thiz->Alarm.AddTask(); + Thiz->GetActorForMe()->Schedule(); + } + }; + + public: + TScheduleActor(TScheduler* scheduler) + : Scheduler(scheduler) + , ScheduleTime(TInstant::Max()) { - } - - /// call Act(TTag) at specified time, unless it is already scheduled at earlier time. - void ScheduleAt(TInstant when) { - TWhatThreadDoesAcquireGuard<TMutex> guard(Mutex, "scheduler: acquiring lock for ScheduleAt"); - - if (when > ScheduleTime) { - // already scheduled + } + + /// call Act(TTag) at specified time, unless it is already scheduled at earlier time. + void ScheduleAt(TInstant when) { + TWhatThreadDoesAcquireGuard<TMutex> guard(Mutex, "scheduler: acquiring lock for ScheduleAt"); + + if (when > ScheduleTime) { + // already scheduled return; } - ScheduleTime = when; - Scheduler->Schedule(new TScheduleItemImpl(GetThis(), when)); + ScheduleTime = when; + Scheduler->Schedule(new TScheduleItemImpl(GetThis(), when)); } - private: - TThis* GetThis() { - return static_cast<TThis*>(this); - } + private: + TThis* GetThis() { + return static_cast<TThis*>(this); + } - TActorForMe* GetActorForMe() { - return static_cast<TActorForMe*>(GetThis()); - } - }; + TActorForMe* GetActorForMe() { + return static_cast<TActorForMe*>(GetThis()); + } + }; } -} +} diff --git a/library/cpp/messagebus/scheduler_actor_ut.cpp b/library/cpp/messagebus/scheduler_actor_ut.cpp index e81ffd3186..e1f1a95f0b 100644 --- a/library/cpp/messagebus/scheduler_actor_ut.cpp +++ b/library/cpp/messagebus/scheduler_actor_ut.cpp @@ -8,15 +8,15 @@ using namespace NBus::NPrivate; using namespace NActor; Y_UNIT_TEST_SUITE(TSchedulerActorTests) { - struct TMyActor: public TAtomicRefCount<TMyActor>, public TActor<TMyActor>, public TScheduleActor<TMyActor> { + struct TMyActor: public TAtomicRefCount<TMyActor>, public TActor<TMyActor>, public TScheduleActor<TMyActor> { TTestSync TestSync; TMyActor(TExecutor* executor, TScheduler* scheduler) : TActor<TMyActor>(executor) , TScheduleActor<TMyActor>(scheduler) , Iteration(0) - { - } + { + } unsigned Iteration; diff --git a/library/cpp/messagebus/session.cpp b/library/cpp/messagebus/session.cpp index 46a7ece6a8..772984a086 100644 --- a/library/cpp/messagebus/session.cpp +++ b/library/cpp/messagebus/session.cpp @@ -5,126 +5,126 @@ using namespace NBus; namespace NBus { - TBusSession::TBusSession() { - } - - //////////////////////////////////////////////////////////////////// - /// \brief Adds peer of connection into connection list - - int CompareByHost(const IRemoteAddr& l, const IRemoteAddr& r) noexcept { - if (l.Addr()->sa_family != r.Addr()->sa_family) { - return l.Addr()->sa_family < r.Addr()->sa_family ? -1 : +1; - } - - switch (l.Addr()->sa_family) { - case AF_INET: { - return memcmp(&(((const sockaddr_in*)l.Addr())->sin_addr), &(((const sockaddr_in*)r.Addr())->sin_addr), sizeof(in_addr)); - } - - case AF_INET6: { - return memcmp(&(((const sockaddr_in6*)l.Addr())->sin6_addr), &(((const sockaddr_in6*)r.Addr())->sin6_addr), sizeof(in6_addr)); - } + TBusSession::TBusSession() { + } + + //////////////////////////////////////////////////////////////////// + /// \brief Adds peer of connection into connection list + + int CompareByHost(const IRemoteAddr& l, const IRemoteAddr& r) noexcept { + if (l.Addr()->sa_family != r.Addr()->sa_family) { + return l.Addr()->sa_family < r.Addr()->sa_family ? -1 : +1; + } + + switch (l.Addr()->sa_family) { + case AF_INET: { + return memcmp(&(((const sockaddr_in*)l.Addr())->sin_addr), &(((const sockaddr_in*)r.Addr())->sin_addr), sizeof(in_addr)); + } + + case AF_INET6: { + return memcmp(&(((const sockaddr_in6*)l.Addr())->sin6_addr), &(((const sockaddr_in6*)r.Addr())->sin6_addr), sizeof(in6_addr)); + } } - return memcmp(l.Addr(), r.Addr(), Min<size_t>(l.Len(), r.Len())); - } - - bool operator<(const TNetAddr& a1, const TNetAddr& a2) { - return CompareByHost(a1, a2) < 0; - } - - size_t TBusSession::GetInFlight(const TNetAddr& addr) const { - size_t r; - GetInFlightBulk({addr}, MakeArrayRef(&r, 1)); - return r; + return memcmp(l.Addr(), r.Addr(), Min<size_t>(l.Len(), r.Len())); } - size_t TBusSession::GetConnectSyscallsNumForTest(const TNetAddr& addr) const { - size_t r; - GetConnectSyscallsNumBulkForTest({addr}, MakeArrayRef(&r, 1)); - return r; - } - - // Split 'host' into name and port taking into account that host can be specified - // as ipv6 address ('[<ipv6 address]:port' notion). - bool SplitHost(const TString& host, TString* hostName, TString* portNum) { - hostName->clear(); - portNum->clear(); - - // Simple check that we have to deal with ipv6 address specification or - // just host name or ipv4 address. + bool operator<(const TNetAddr& a1, const TNetAddr& a2) { + return CompareByHost(a1, a2) < 0; + } + + size_t TBusSession::GetInFlight(const TNetAddr& addr) const { + size_t r; + GetInFlightBulk({addr}, MakeArrayRef(&r, 1)); + return r; + } + + size_t TBusSession::GetConnectSyscallsNumForTest(const TNetAddr& addr) const { + size_t r; + GetConnectSyscallsNumBulkForTest({addr}, MakeArrayRef(&r, 1)); + return r; + } + + // Split 'host' into name and port taking into account that host can be specified + // as ipv6 address ('[<ipv6 address]:port' notion). + bool SplitHost(const TString& host, TString* hostName, TString* portNum) { + hostName->clear(); + portNum->clear(); + + // Simple check that we have to deal with ipv6 address specification or + // just host name or ipv4 address. if (!host.empty() && (host[0] == '[')) { - size_t pos = host.find(']'); - if (pos < 2 || pos == TString::npos) { - // '[]' and '[<address>' are errors. - return false; + size_t pos = host.find(']'); + if (pos < 2 || pos == TString::npos) { + // '[]' and '[<address>' are errors. + return false; + } + + *hostName = host.substr(1, pos - 1); + + pos++; + if (pos != host.length()) { + if (host[pos] != ':') { + // Do not allow '[...]a' but '[...]:' is ok (as for ipv4 before + return false; + } + + *portNum = host.substr(pos + 1); } - - *hostName = host.substr(1, pos - 1); - - pos++; - if (pos != host.length()) { - if (host[pos] != ':') { - // Do not allow '[...]a' but '[...]:' is ok (as for ipv4 before - return false; - } - - *portNum = host.substr(pos + 1); - } - } else { - size_t pos = host.find(':'); - if (pos != TString::npos) { - if (pos == 0) { - // Treat ':<port>' as errors but allow or '<host>:' for compatibility. - return false; - } - - *portNum = host.substr(pos + 1); + } else { + size_t pos = host.find(':'); + if (pos != TString::npos) { + if (pos == 0) { + // Treat ':<port>' as errors but allow or '<host>:' for compatibility. + return false; + } + + *portNum = host.substr(pos + 1); } - *hostName = host.substr(0, pos); + *hostName = host.substr(0, pos); } - return true; + return true; } - /// registers external session on host:port with locator service - int TBusSession::RegisterService(const char* host, TBusKey start /*= YBUS_KEYMIN*/, TBusKey end /*= YBUS_KEYMAX*/, EIpVersion ipVersion) { - TString hostName; - TString port; - int portNum; + /// registers external session on host:port with locator service + int TBusSession::RegisterService(const char* host, TBusKey start /*= YBUS_KEYMIN*/, TBusKey end /*= YBUS_KEYMAX*/, EIpVersion ipVersion) { + TString hostName; + TString port; + int portNum; - if (!SplitHost(host, &hostName, &port)) { - hostName = host; - } + if (!SplitHost(host, &hostName, &port)) { + hostName = host; + } if (port.empty()) { - portNum = GetProto()->GetPort(); - } else { - try { - portNum = FromString<int>(port); - } catch (const TFromStringException&) { - return -1; - } - } - - TBusService service = GetProto()->GetService(); + portNum = GetProto()->GetPort(); + } else { + try { + portNum = FromString<int>(port); + } catch (const TFromStringException&) { + return -1; + } + } + + TBusService service = GetProto()->GetService(); return GetQueue()->GetLocator()->Register(service, hostName.data(), portNum, start, end, ipVersion); } - TBusSession::~TBusSession() { + TBusSession::~TBusSession() { } } -TBusClientSessionPtr TBusClientSession::Create(TBusProtocol* proto, IBusClientHandler* handler, const TBusClientSessionConfig& config, TBusMessageQueuePtr queue) { +TBusClientSessionPtr TBusClientSession::Create(TBusProtocol* proto, IBusClientHandler* handler, const TBusClientSessionConfig& config, TBusMessageQueuePtr queue) { return queue->CreateSource(proto, handler, config); } -TBusServerSessionPtr TBusServerSession::Create(TBusProtocol* proto, IBusServerHandler* handler, const TBusServerSessionConfig& config, TBusMessageQueuePtr queue) { +TBusServerSessionPtr TBusServerSession::Create(TBusProtocol* proto, IBusServerHandler* handler, const TBusServerSessionConfig& config, TBusMessageQueuePtr queue) { return queue->CreateDestination(proto, handler, config); } -TBusServerSessionPtr TBusServerSession::Create(TBusProtocol* proto, IBusServerHandler* handler, const TBusServerSessionConfig& config, TBusMessageQueuePtr queue, const TVector<TBindResult>& bindTo) { +TBusServerSessionPtr TBusServerSession::Create(TBusProtocol* proto, IBusServerHandler* handler, const TBusServerSessionConfig& config, TBusMessageQueuePtr queue, const TVector<TBindResult>& bindTo) { return queue->CreateDestination(proto, handler, config, bindTo); } diff --git a/library/cpp/messagebus/session.h b/library/cpp/messagebus/session.h index fb12ab7c22..857f58d7e5 100644 --- a/library/cpp/messagebus/session.h +++ b/library/cpp/messagebus/session.h @@ -15,211 +15,211 @@ #include <util/generic/ptr.h> namespace NBus { - template <typename TBusSessionSubclass> - class TBusSessionPtr; - using TBusClientSessionPtr = TBusSessionPtr<TBusClientSession>; - using TBusServerSessionPtr = TBusSessionPtr<TBusServerSession>; - - /////////////////////////////////////////////////////////////////// - /// \brief Interface of session object. - - /// Each client and server - /// should instantiate session object to be able to communicate via bus - /// client: sess = queue->CreateSource(protocol, handler); - /// server: sess = queue->CreateDestination(protocol, handler); - - class TBusSession: public TWeakRefCounted<TBusSession> { - public: - size_t GetInFlight(const TNetAddr& addr) const; - size_t GetConnectSyscallsNumForTest(const TNetAddr& addr) const; - - virtual void GetInFlightBulk(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const = 0; - virtual void GetConnectSyscallsNumBulkForTest(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const = 0; - - virtual int GetInFlight() const noexcept = 0; - /// monitoring status of current session and it's connections - virtual TString GetStatus(ui16 flags = YBUS_STATUS_CONNS) = 0; - virtual TConnectionStatusMonRecord GetStatusProtobuf() = 0; - virtual NPrivate::TSessionDumpStatus GetStatusRecordInternal() = 0; - virtual TString GetStatusSingleLine() = 0; - /// return session config - virtual const TBusSessionConfig* GetConfig() const noexcept = 0; - /// return session protocol - virtual const TBusProtocol* GetProto() const noexcept = 0; - virtual TBusMessageQueue* GetQueue() const noexcept = 0; - - /// registers external session on host:port with locator service - int RegisterService(const char* hostname, TBusKey start = YBUS_KEYMIN, TBusKey end = YBUS_KEYMAX, EIpVersion ipVersion = EIP_VERSION_4); - - protected: - TBusSession(); - - public: - virtual TString GetNameInternal() = 0; - - virtual void Shutdown() = 0; - - virtual ~TBusSession(); - }; - - struct TBusClientSession: public virtual TBusSession { - typedef ::NBus::NPrivate::TRemoteClientSession TImpl; - - static TBusClientSessionPtr Create( - TBusProtocol* proto, - IBusClientHandler* handler, + template <typename TBusSessionSubclass> + class TBusSessionPtr; + using TBusClientSessionPtr = TBusSessionPtr<TBusClientSession>; + using TBusServerSessionPtr = TBusSessionPtr<TBusServerSession>; + + /////////////////////////////////////////////////////////////////// + /// \brief Interface of session object. + + /// Each client and server + /// should instantiate session object to be able to communicate via bus + /// client: sess = queue->CreateSource(protocol, handler); + /// server: sess = queue->CreateDestination(protocol, handler); + + class TBusSession: public TWeakRefCounted<TBusSession> { + public: + size_t GetInFlight(const TNetAddr& addr) const; + size_t GetConnectSyscallsNumForTest(const TNetAddr& addr) const; + + virtual void GetInFlightBulk(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const = 0; + virtual void GetConnectSyscallsNumBulkForTest(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const = 0; + + virtual int GetInFlight() const noexcept = 0; + /// monitoring status of current session and it's connections + virtual TString GetStatus(ui16 flags = YBUS_STATUS_CONNS) = 0; + virtual TConnectionStatusMonRecord GetStatusProtobuf() = 0; + virtual NPrivate::TSessionDumpStatus GetStatusRecordInternal() = 0; + virtual TString GetStatusSingleLine() = 0; + /// return session config + virtual const TBusSessionConfig* GetConfig() const noexcept = 0; + /// return session protocol + virtual const TBusProtocol* GetProto() const noexcept = 0; + virtual TBusMessageQueue* GetQueue() const noexcept = 0; + + /// registers external session on host:port with locator service + int RegisterService(const char* hostname, TBusKey start = YBUS_KEYMIN, TBusKey end = YBUS_KEYMAX, EIpVersion ipVersion = EIP_VERSION_4); + + protected: + TBusSession(); + + public: + virtual TString GetNameInternal() = 0; + + virtual void Shutdown() = 0; + + virtual ~TBusSession(); + }; + + struct TBusClientSession: public virtual TBusSession { + typedef ::NBus::NPrivate::TRemoteClientSession TImpl; + + static TBusClientSessionPtr Create( + TBusProtocol* proto, + IBusClientHandler* handler, const TBusClientSessionConfig& config, TBusMessageQueuePtr queue); - virtual TBusClientConnectionPtr GetConnection(const TNetAddr&) = 0; + virtual TBusClientConnectionPtr GetConnection(const TNetAddr&) = 0; - /// if you want to open connection early - virtual void OpenConnection(const TNetAddr&) = 0; + /// if you want to open connection early + virtual void OpenConnection(const TNetAddr&) = 0; - /// Send message to the destination - /// If addr is set then use it as destination. - /// Takes ownership of addr (see ClearState method). - virtual EMessageStatus SendMessage(TBusMessage* pMes, const TNetAddr* addr = nullptr, bool wait = false) = 0; + /// Send message to the destination + /// If addr is set then use it as destination. + /// Takes ownership of addr (see ClearState method). + virtual EMessageStatus SendMessage(TBusMessage* pMes, const TNetAddr* addr = nullptr, bool wait = false) = 0; - virtual EMessageStatus SendMessageOneWay(TBusMessage* pMes, const TNetAddr* addr = nullptr, bool wait = false) = 0; + virtual EMessageStatus SendMessageOneWay(TBusMessage* pMes, const TNetAddr* addr = nullptr, bool wait = false) = 0; - /// Like SendMessage but cares about message - template <typename T /* <: TBusMessage */> - EMessageStatus SendMessageAutoPtr(const TAutoPtr<T>& mes, const TNetAddr* addr = nullptr, bool wait = false) { - EMessageStatus status = SendMessage(mes.Get(), addr, wait); - if (status == MESSAGE_OK) + /// Like SendMessage but cares about message + template <typename T /* <: TBusMessage */> + EMessageStatus SendMessageAutoPtr(const TAutoPtr<T>& mes, const TNetAddr* addr = nullptr, bool wait = false) { + EMessageStatus status = SendMessage(mes.Get(), addr, wait); + if (status == MESSAGE_OK) Y_UNUSED(mes.Release()); - return status; - } - - /// Like SendMessageOneWay but cares about message - template <typename T /* <: TBusMessage */> - EMessageStatus SendMessageOneWayAutoPtr(const TAutoPtr<T>& mes, const TNetAddr* addr = nullptr, bool wait = false) { - EMessageStatus status = SendMessageOneWay(mes.Get(), addr, wait); - if (status == MESSAGE_OK) + return status; + } + + /// Like SendMessageOneWay but cares about message + template <typename T /* <: TBusMessage */> + EMessageStatus SendMessageOneWayAutoPtr(const TAutoPtr<T>& mes, const TNetAddr* addr = nullptr, bool wait = false) { + EMessageStatus status = SendMessageOneWay(mes.Get(), addr, wait); + if (status == MESSAGE_OK) Y_UNUSED(mes.Release()); - return status; - } + return status; + } - EMessageStatus SendMessageMove(TBusMessageAutoPtr message, const TNetAddr* addr = nullptr, bool wait = false) { - return SendMessageAutoPtr(message, addr, wait); - } + EMessageStatus SendMessageMove(TBusMessageAutoPtr message, const TNetAddr* addr = nullptr, bool wait = false) { + return SendMessageAutoPtr(message, addr, wait); + } - EMessageStatus SendMessageOneWayMove(TBusMessageAutoPtr message, const TNetAddr* addr = nullptr, bool wait = false) { - return SendMessageOneWayAutoPtr(message, addr, wait); - } + EMessageStatus SendMessageOneWayMove(TBusMessageAutoPtr message, const TNetAddr* addr = nullptr, bool wait = false) { + return SendMessageOneWayAutoPtr(message, addr, wait); + } - // TODO: implement similar one-way methods - }; + // TODO: implement similar one-way methods + }; - struct TBusServerSession: public virtual TBusSession { - typedef ::NBus::NPrivate::TRemoteServerSession TImpl; + struct TBusServerSession: public virtual TBusSession { + typedef ::NBus::NPrivate::TRemoteServerSession TImpl; - static TBusServerSessionPtr Create( - TBusProtocol* proto, - IBusServerHandler* handler, + static TBusServerSessionPtr Create( + TBusProtocol* proto, + IBusServerHandler* handler, const TBusServerSessionConfig& config, TBusMessageQueuePtr queue); - static TBusServerSessionPtr Create( - TBusProtocol* proto, - IBusServerHandler* handler, + static TBusServerSessionPtr Create( + TBusProtocol* proto, + IBusServerHandler* handler, const TBusServerSessionConfig& config, TBusMessageQueuePtr queue, const TVector<TBindResult>& bindTo); - // TODO: make parameter non-const - virtual EMessageStatus SendReply(const TBusIdentity& ident, TBusMessage* pRep) = 0; + // TODO: make parameter non-const + virtual EMessageStatus SendReply(const TBusIdentity& ident, TBusMessage* pRep) = 0; - // TODO: make parameter non-const - virtual EMessageStatus ForgetRequest(const TBusIdentity& ident) = 0; + // TODO: make parameter non-const + virtual EMessageStatus ForgetRequest(const TBusIdentity& ident) = 0; - template <typename U /* <: TBusMessage */> - EMessageStatus SendReplyAutoPtr(TBusIdentity& ident, TAutoPtr<U>& resp) { - EMessageStatus status = SendReply(const_cast<const TBusIdentity&>(ident), resp.Get()); - if (status == MESSAGE_OK) { + template <typename U /* <: TBusMessage */> + EMessageStatus SendReplyAutoPtr(TBusIdentity& ident, TAutoPtr<U>& resp) { + EMessageStatus status = SendReply(const_cast<const TBusIdentity&>(ident), resp.Get()); + if (status == MESSAGE_OK) { Y_UNUSED(resp.Release()); - } - return status; + } + return status; } - EMessageStatus SendReplyMove(TBusIdentity& ident, TBusMessageAutoPtr resp) { - return SendReplyAutoPtr(ident, resp); - } - - /// Pause input from the network. - /// It is valid to call this method in parallel. - /// TODO: pull this method up to TBusSession. - virtual void PauseInput(bool pause) = 0; - virtual unsigned GetActualListenPort() = 0; - }; - - namespace NPrivate { - template <typename TBusSessionSubclass> - class TBusOwnerSessionPtr: public TAtomicRefCount<TBusOwnerSessionPtr<TBusSessionSubclass>> { - private: - TIntrusivePtr<TBusSessionSubclass> Ptr; - - public: - TBusOwnerSessionPtr(TBusSessionSubclass* session) - : Ptr(session) - { - Y_ASSERT(!!Ptr); - } - - ~TBusOwnerSessionPtr() { - Ptr->Shutdown(); - } - - TBusSessionSubclass* Get() const { - return reinterpret_cast<TBusSessionSubclass*>(Ptr.Get()); - } - }; + EMessageStatus SendReplyMove(TBusIdentity& ident, TBusMessageAutoPtr resp) { + return SendReplyAutoPtr(ident, resp); + } + + /// Pause input from the network. + /// It is valid to call this method in parallel. + /// TODO: pull this method up to TBusSession. + virtual void PauseInput(bool pause) = 0; + virtual unsigned GetActualListenPort() = 0; + }; + + namespace NPrivate { + template <typename TBusSessionSubclass> + class TBusOwnerSessionPtr: public TAtomicRefCount<TBusOwnerSessionPtr<TBusSessionSubclass>> { + private: + TIntrusivePtr<TBusSessionSubclass> Ptr; + + public: + TBusOwnerSessionPtr(TBusSessionSubclass* session) + : Ptr(session) + { + Y_ASSERT(!!Ptr); + } + + ~TBusOwnerSessionPtr() { + Ptr->Shutdown(); + } + + TBusSessionSubclass* Get() const { + return reinterpret_cast<TBusSessionSubclass*>(Ptr.Get()); + } + }; } - template <typename TBusSessionSubclass> - class TBusSessionPtr { - private: - TIntrusivePtr<NPrivate::TBusOwnerSessionPtr<TBusSessionSubclass>> SmartPtr; - TBusSessionSubclass* Ptr; - - public: - TBusSessionPtr() - : Ptr() - { - } - TBusSessionPtr(TBusSessionSubclass* session) - : SmartPtr(!!session ? new NPrivate::TBusOwnerSessionPtr<TBusSessionSubclass>(session) : nullptr) - , Ptr(session) - { - } - - TBusSessionSubclass* Get() const { - return Ptr; - } - operator TBusSessionSubclass*() { - return Get(); - } - TBusSessionSubclass& operator*() const { - return *Get(); - } - TBusSessionSubclass* operator->() const { - return Get(); - } - - bool operator!() const { - return !Ptr; - } - - void Swap(TBusSessionPtr& t) noexcept { - DoSwap(SmartPtr, t.SmartPtr); - DoSwap(Ptr, t.Ptr); - } - - void Drop() { - TBusSessionPtr().Swap(*this); - } - }; + template <typename TBusSessionSubclass> + class TBusSessionPtr { + private: + TIntrusivePtr<NPrivate::TBusOwnerSessionPtr<TBusSessionSubclass>> SmartPtr; + TBusSessionSubclass* Ptr; + + public: + TBusSessionPtr() + : Ptr() + { + } + TBusSessionPtr(TBusSessionSubclass* session) + : SmartPtr(!!session ? new NPrivate::TBusOwnerSessionPtr<TBusSessionSubclass>(session) : nullptr) + , Ptr(session) + { + } + + TBusSessionSubclass* Get() const { + return Ptr; + } + operator TBusSessionSubclass*() { + return Get(); + } + TBusSessionSubclass& operator*() const { + return *Get(); + } + TBusSessionSubclass* operator->() const { + return Get(); + } + + bool operator!() const { + return !Ptr; + } + + void Swap(TBusSessionPtr& t) noexcept { + DoSwap(SmartPtr, t.SmartPtr); + DoSwap(Ptr, t.Ptr); + } + + void Drop() { + TBusSessionPtr().Swap(*this); + } + }; } diff --git a/library/cpp/messagebus/session_impl.cpp b/library/cpp/messagebus/session_impl.cpp index ddf9f360c4..cff202c5b8 100644 --- a/library/cpp/messagebus/session_impl.cpp +++ b/library/cpp/messagebus/session_impl.cpp @@ -40,10 +40,10 @@ namespace { } TConnectionsAcceptorsSnapshot::TConnectionsAcceptorsSnapshot() - : LastConnectionId(0) - , LastAcceptorId(0) -{ -} + : LastConnectionId(0) + , LastAcceptorId(0) +{ +} struct TBusSessionImpl::TImpl { TRemoteConnectionWriterIncrementalStatus DeadConnectionWriterStatusSummary; @@ -62,7 +62,7 @@ namespace { copy.TotalTimeout = config.SendTimeout + TDuration::MilliSeconds(10).MilliSeconds(); } else if (copy.SendTimeout == 0) { Y_ASSERT(copy.TotalTimeout != 0); - if ((ui64)copy.TotalTimeout > (ui64)TDuration::MilliSeconds(10).MilliSeconds()) { + if ((ui64)copy.TotalTimeout > (ui64)TDuration::MilliSeconds(10).MilliSeconds()) { copy.SendTimeout = copy.TotalTimeout - TDuration::MilliSeconds(10).MilliSeconds(); } else { copy.SendTimeout = copy.TotalTimeout; @@ -90,8 +90,8 @@ namespace { } TBusSessionImpl::TBusSessionImpl(bool isSource, TBusMessageQueue* queue, TBusProtocol* proto, - IBusErrorHandler* handler, - const TBusSessionConfig& config, const TString& name) + IBusErrorHandler* handler, + const TBusSessionConfig& config, const TString& name) : TActor<TBusSessionImpl, TStatusTag>(queue->WorkQueue.Get()) , TActor<TBusSessionImpl, TConnectionTag>(queue->WorkQueue.Get()) , Impl(new TImpl) @@ -111,7 +111,7 @@ TBusSessionImpl::TBusSessionImpl(bool isSource, TBusMessageQueue* queue, TBusPro Impl->DeadAcceptorStatusSummary.Summary = true; ReadEventLoopThread.Reset(new NThreading::TLegacyFuture<void, false>(std::bind(&TEventLoop::Run, std::ref(ReadEventLoop)))); - WriteEventLoopThread.Reset(new NThreading::TLegacyFuture<void, false>(std::bind(&TEventLoop::Run, std::ref(WriteEventLoop)))); + WriteEventLoopThread.Reset(new NThreading::TLegacyFuture<void, false>(std::bind(&TEventLoop::Run, std::ref(WriteEventLoop)))); Queue->Schedule(IScheduleItemAutoPtr(new TScheduleSession(this, TInstant::Now() + Config.Secret.TimeoutPeriod))); } @@ -127,8 +127,8 @@ TBusSessionStatus::TBusSessionStatus() : InFlightCount(0) , InFlightSize(0) , InputPaused(false) -{ -} +{ +} void TBusSessionImpl::Shutdown() { if (!AtomicCas(&Down, 1, 0)) { @@ -233,7 +233,7 @@ void TBusSessionImpl::FillStatus() { TSessionDumpStatus TBusSessionImpl::GetStatusRecordInternal() { // Probably useless, because it returns cached info now Y_VERIFY(!Queue->GetExecutor()->IsInExecutorThread(), - "GetStatus must not be called from executor thread"); + "GetStatus must not be called from executor thread"); TGuard<TMutex> guard(StatusData.StatusDumpCachedMutex); // TODO: returns zeros for a second after start @@ -247,9 +247,9 @@ TString TBusSessionImpl::GetStatus(ui16 flags) { return GetStatusRecordInternal().PrintToString(); } -TConnectionStatusMonRecord TBusSessionImpl::GetStatusProtobuf() { +TConnectionStatusMonRecord TBusSessionImpl::GetStatusProtobuf() { Y_VERIFY(!Queue->GetExecutor()->IsInExecutorThread(), - "GetStatus must not be called from executor thread"); + "GetStatus must not be called from executor thread"); TGuard<TMutex> guard(StatusData.StatusDumpCachedMutex); @@ -289,8 +289,8 @@ void TBusSessionImpl::ProcessItem(TConnectionTag, ::NActor::TDefaultTag, const T } //if (Connections.find(addr) != Connections.end()) { - // TODO: it is possible - // won't be a problem after socket address replaced with id + // TODO: it is possible + // won't be a problem after socket address replaced with id //} TRemoteConnectionPtr c(new TRemoteServerConnection(VerifyDynamicCast<TRemoteServerSession*>(this), ++LastConnectionId, onAccept.addr)); @@ -316,14 +316,14 @@ void TBusSessionImpl::ProcessItem(TConnectionTag, TRemoveTag, TRemoteConnectionP SendSnapshotToStatusActor(); } -void TBusSessionImpl::ProcessConnectionsAcceptorsShapshotQueueItem(TAtomicSharedPtr<TConnectionsAcceptorsSnapshot> snapshot) { +void TBusSessionImpl::ProcessConnectionsAcceptorsShapshotQueueItem(TAtomicSharedPtr<TConnectionsAcceptorsSnapshot> snapshot) { for (TVector<TRemoteConnectionPtr>::const_iterator connection = snapshot->Connections.begin(); - connection != snapshot->Connections.end(); ++connection) { + connection != snapshot->Connections.end(); ++connection) { Y_ASSERT((*connection)->ConnectionId <= snapshot->LastConnectionId); } for (TVector<TAcceptorPtr>::const_iterator acceptor = snapshot->Acceptors.begin(); - acceptor != snapshot->Acceptors.end(); ++acceptor) { + acceptor != snapshot->Acceptors.end(); ++acceptor) { Y_ASSERT((*acceptor)->AcceptorId <= snapshot->LastAcceptorId); } @@ -388,7 +388,7 @@ void TBusSessionImpl::StatusUpdateCachedDump() { TStringStream ss; for (TVector<TAcceptorPtr>::const_iterator acceptor = acceptors.begin(); - acceptor != acceptors.end(); ++acceptor) { + acceptor != acceptors.end(); ++acceptor) { const TAcceptorStatus status = (*acceptor)->GranStatus.Listen.Get(); acceptorStatusSummary += status; @@ -406,7 +406,7 @@ void TBusSessionImpl::StatusUpdateCachedDump() { TStringStream ss; for (TVector<TRemoteConnectionPtr>::const_iterator connection = connections.begin(); - connection != connections.end(); ++connection) { + connection != connections.end(); ++connection) { if (connection != connections.begin()) { ss << "\n"; } @@ -434,8 +434,8 @@ void TBusSessionImpl::StatusUpdateCachedDump() { TBusSessionImpl::TStatusData::TStatusData() : ConnectionsAcceptorsSnapshot(new TConnectionsAcceptorsSnapshot) -{ -} +{ +} void TBusSessionImpl::Act(TStatusTag) { TInstant now = TInstant::Now(); @@ -463,8 +463,8 @@ void TBusSessionImpl::Act(TStatusTag) { StatusUpdateCachedDumpIfNecessary(now); } -TBusSessionImpl::TConnectionsData::TConnectionsData() { -} +TBusSessionImpl::TConnectionsData::TConnectionsData() { +} void TBusSessionImpl::Act(TConnectionTag) { TConnectionsGuard guard(ConnectionsLock); @@ -487,11 +487,11 @@ void TBusSessionImpl::Listen(int port, TBusMessageQueue* q) { Listen(BindOnPort(port, Config.ReusePort).second, q); } -void TBusSessionImpl::Listen(const TVector<TBindResult>& bindTo, TBusMessageQueue* q) { +void TBusSessionImpl::Listen(const TVector<TBindResult>& bindTo, TBusMessageQueue* q) { Y_ASSERT(q == Queue); int actualPort = -1; - for (const TBindResult& br : bindTo) { + for (const TBindResult& br : bindTo) { if (actualPort == -1) { actualPort = br.Addr.GetPort(); } else { @@ -513,7 +513,7 @@ void TBusSessionImpl::Listen(const TVector<TBindResult>& bindTo, TBusMessageQueu void TBusSessionImpl::SendSnapshotToStatusActor() { //Y_ASSERT(ConnectionsLock.IsLocked()); - TAtomicSharedPtr<TConnectionsAcceptorsSnapshot> snapshot(new TConnectionsAcceptorsSnapshot); + TAtomicSharedPtr<TConnectionsAcceptorsSnapshot> snapshot(new TConnectionsAcceptorsSnapshot); GetAcceptorsLockAquired(&snapshot->Acceptors); GetConnectionsLockAquired(&snapshot->Connections); snapshot->LastAcceptorId = LastAcceptorId; diff --git a/library/cpp/messagebus/session_impl.h b/library/cpp/messagebus/session_impl.h index 90ef246ff8..e099c798b9 100644 --- a/library/cpp/messagebus/session_impl.h +++ b/library/cpp/messagebus/session_impl.h @@ -19,241 +19,241 @@ #include <util/generic/array_ref.h> #include <util/generic/string.h> -namespace NBus { - namespace NPrivate { - typedef TIntrusivePtr<TRemoteClientConnection> TRemoteClientConnectionPtr; - typedef TIntrusivePtr<TRemoteServerConnection> TRemoteServerConnectionPtr; - - typedef TIntrusivePtr<TRemoteServerSession> TRemoteServerSessionPtr; - - typedef TIntrusivePtr<TAcceptor> TAcceptorPtr; - typedef TVector<TAcceptorPtr> TAcceptorsPtrs; - - struct TConnectionsAcceptorsSnapshot { - TVector<TRemoteConnectionPtr> Connections; - TVector<TAcceptorPtr> Acceptors; - ui64 LastConnectionId; - ui64 LastAcceptorId; - - TConnectionsAcceptorsSnapshot(); - }; - - typedef TAtomicSharedPtr<TConnectionsAcceptorsSnapshot> TConnectionsAcceptorsSnapshotPtr; - - struct TOnAccept { - SOCKET s; - TNetAddr addr; - TInstant now; - }; - - struct TStatusTag {}; - struct TConnectionTag {}; - - struct TDeadConnectionTag {}; - struct TRemoveTag {}; - - struct TBusSessionImpl - : public virtual TBusSession, - private ::NActor::TActor<TBusSessionImpl, TStatusTag>, - private ::NActor::TActor<TBusSessionImpl, TConnectionTag> - - , - private ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionWriterIncrementalStatus, TStatusTag, TDeadConnectionTag>, - private ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionReaderIncrementalStatus, TStatusTag, TDeadConnectionTag>, - private ::NActor::TQueueInActor<TBusSessionImpl, TAcceptorStatus, TStatusTag, TDeadConnectionTag> - - , - private ::NActor::TQueueInActor<TBusSessionImpl, TOnAccept, TConnectionTag>, - private ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionPtr, TConnectionTag, TRemoveTag> { - friend class TAcceptor; - friend class TRemoteConnection; - friend class TRemoteServerConnection; - friend class ::NActor::TActor<TBusSessionImpl, TStatusTag>; - friend class ::NActor::TActor<TBusSessionImpl, TConnectionTag>; - friend class ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionWriterIncrementalStatus, TStatusTag, TDeadConnectionTag>; - friend class ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionReaderIncrementalStatus, TStatusTag, TDeadConnectionTag>; - friend class ::NActor::TQueueInActor<TBusSessionImpl, TAcceptorStatus, TStatusTag, TDeadConnectionTag>; - friend class ::NActor::TQueueInActor<TBusSessionImpl, TOnAccept, TConnectionTag>; - friend class ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionPtr, TConnectionTag, TRemoveTag>; - - public: - ::NActor::TQueueInActor<TBusSessionImpl, TOnAccept, TConnectionTag>* GetOnAcceptQueue() { - return this; - } +namespace NBus { + namespace NPrivate { + typedef TIntrusivePtr<TRemoteClientConnection> TRemoteClientConnectionPtr; + typedef TIntrusivePtr<TRemoteServerConnection> TRemoteServerConnectionPtr; + + typedef TIntrusivePtr<TRemoteServerSession> TRemoteServerSessionPtr; + + typedef TIntrusivePtr<TAcceptor> TAcceptorPtr; + typedef TVector<TAcceptorPtr> TAcceptorsPtrs; + + struct TConnectionsAcceptorsSnapshot { + TVector<TRemoteConnectionPtr> Connections; + TVector<TAcceptorPtr> Acceptors; + ui64 LastConnectionId; + ui64 LastAcceptorId; + + TConnectionsAcceptorsSnapshot(); + }; + + typedef TAtomicSharedPtr<TConnectionsAcceptorsSnapshot> TConnectionsAcceptorsSnapshotPtr; + + struct TOnAccept { + SOCKET s; + TNetAddr addr; + TInstant now; + }; + + struct TStatusTag {}; + struct TConnectionTag {}; + + struct TDeadConnectionTag {}; + struct TRemoveTag {}; + + struct TBusSessionImpl + : public virtual TBusSession, + private ::NActor::TActor<TBusSessionImpl, TStatusTag>, + private ::NActor::TActor<TBusSessionImpl, TConnectionTag> + + , + private ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionWriterIncrementalStatus, TStatusTag, TDeadConnectionTag>, + private ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionReaderIncrementalStatus, TStatusTag, TDeadConnectionTag>, + private ::NActor::TQueueInActor<TBusSessionImpl, TAcceptorStatus, TStatusTag, TDeadConnectionTag> + + , + private ::NActor::TQueueInActor<TBusSessionImpl, TOnAccept, TConnectionTag>, + private ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionPtr, TConnectionTag, TRemoveTag> { + friend class TAcceptor; + friend class TRemoteConnection; + friend class TRemoteServerConnection; + friend class ::NActor::TActor<TBusSessionImpl, TStatusTag>; + friend class ::NActor::TActor<TBusSessionImpl, TConnectionTag>; + friend class ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionWriterIncrementalStatus, TStatusTag, TDeadConnectionTag>; + friend class ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionReaderIncrementalStatus, TStatusTag, TDeadConnectionTag>; + friend class ::NActor::TQueueInActor<TBusSessionImpl, TAcceptorStatus, TStatusTag, TDeadConnectionTag>; + friend class ::NActor::TQueueInActor<TBusSessionImpl, TOnAccept, TConnectionTag>; + friend class ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionPtr, TConnectionTag, TRemoveTag>; + + public: + ::NActor::TQueueInActor<TBusSessionImpl, TOnAccept, TConnectionTag>* GetOnAcceptQueue() { + return this; + } - ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionPtr, TConnectionTag, TRemoveTag>* GetRemoveConnectionQueue() { - return this; - } - - ::NActor::TActor<TBusSessionImpl, TConnectionTag>* GetConnectionActor() { - return this; - } - - typedef TGuard<TMutex> TConnectionsGuard; - - TBusSessionImpl(bool isSource, TBusMessageQueue* queue, TBusProtocol* proto, - IBusErrorHandler* handler, - const TBusSessionConfig& config, const TString& name); - - ~TBusSessionImpl() override; + ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionPtr, TConnectionTag, TRemoveTag>* GetRemoveConnectionQueue() { + return this; + } + + ::NActor::TActor<TBusSessionImpl, TConnectionTag>* GetConnectionActor() { + return this; + } + + typedef TGuard<TMutex> TConnectionsGuard; + + TBusSessionImpl(bool isSource, TBusMessageQueue* queue, TBusProtocol* proto, + IBusErrorHandler* handler, + const TBusSessionConfig& config, const TString& name); + + ~TBusSessionImpl() override; - void Shutdown() override; - bool IsDown(); + void Shutdown() override; + bool IsDown(); - size_t GetInFlightImpl(const TNetAddr& addr) const; - size_t GetConnectSyscallsNumForTestImpl(const TNetAddr& addr) const; + size_t GetInFlightImpl(const TNetAddr& addr) const; + size_t GetConnectSyscallsNumForTestImpl(const TNetAddr& addr) const; - void GetInFlightBulk(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const override; - void GetConnectSyscallsNumBulkForTest(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const override; + void GetInFlightBulk(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const override; + void GetConnectSyscallsNumBulkForTest(TArrayRef<const TNetAddr> addrs, TArrayRef<size_t> results) const override; - virtual void FillStatus(); - TSessionDumpStatus GetStatusRecordInternal() override; - TString GetStatus(ui16 flags = YBUS_STATUS_CONNS) override; - TConnectionStatusMonRecord GetStatusProtobuf() override; - TString GetStatusSingleLine() override; + virtual void FillStatus(); + TSessionDumpStatus GetStatusRecordInternal() override; + TString GetStatus(ui16 flags = YBUS_STATUS_CONNS) override; + TConnectionStatusMonRecord GetStatusProtobuf() override; + TString GetStatusSingleLine() override; - void ProcessItem(TStatusTag, TDeadConnectionTag, const TRemoteConnectionWriterIncrementalStatus&); - void ProcessItem(TStatusTag, TDeadConnectionTag, const TRemoteConnectionReaderIncrementalStatus&); - void ProcessItem(TStatusTag, TDeadConnectionTag, const TAcceptorStatus&); - void ProcessItem(TStatusTag, ::NActor::TDefaultTag, const TAcceptorStatus&); - void ProcessItem(TConnectionTag, ::NActor::TDefaultTag, const TOnAccept&); - void ProcessItem(TConnectionTag, TRemoveTag, TRemoteConnectionPtr); - void ProcessConnectionsAcceptorsShapshotQueueItem(TAtomicSharedPtr<TConnectionsAcceptorsSnapshot>); - void StatusUpdateCachedDump(); - void StatusUpdateCachedDumpIfNecessary(TInstant now); - void Act(TStatusTag); - void Act(TConnectionTag); + void ProcessItem(TStatusTag, TDeadConnectionTag, const TRemoteConnectionWriterIncrementalStatus&); + void ProcessItem(TStatusTag, TDeadConnectionTag, const TRemoteConnectionReaderIncrementalStatus&); + void ProcessItem(TStatusTag, TDeadConnectionTag, const TAcceptorStatus&); + void ProcessItem(TStatusTag, ::NActor::TDefaultTag, const TAcceptorStatus&); + void ProcessItem(TConnectionTag, ::NActor::TDefaultTag, const TOnAccept&); + void ProcessItem(TConnectionTag, TRemoveTag, TRemoteConnectionPtr); + void ProcessConnectionsAcceptorsShapshotQueueItem(TAtomicSharedPtr<TConnectionsAcceptorsSnapshot>); + void StatusUpdateCachedDump(); + void StatusUpdateCachedDumpIfNecessary(TInstant now); + void Act(TStatusTag); + void Act(TConnectionTag); - TBusProtocol* GetProto() const noexcept override; - const TBusSessionConfig* GetConfig() const noexcept override; - TBusMessageQueue* GetQueue() const noexcept override; - TString GetNameInternal() override; - - virtual void OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& newMsg) = 0; - - void Listen(int port, TBusMessageQueue* q); - void Listen(const TVector<TBindResult>& bindTo, TBusMessageQueue* q); - TBusConnection* Accept(SOCKET listen); - - inline ::NActor::TActor<TBusSessionImpl, TStatusTag>* GetStatusActor() { - return this; - } - inline ::NActor::TActor<TBusSessionImpl, TConnectionTag>* GetConnectionsActor() { - return this; - } - - typedef THashMap<TBusSocketAddr, TRemoteConnectionPtr> TAddrRemoteConnections; - - void SendSnapshotToStatusActor(); + TBusProtocol* GetProto() const noexcept override; + const TBusSessionConfig* GetConfig() const noexcept override; + TBusMessageQueue* GetQueue() const noexcept override; + TString GetNameInternal() override; + + virtual void OnMessageReceived(TRemoteConnection* c, TVectorSwaps<TBusMessagePtrAndHeader>& newMsg) = 0; + + void Listen(int port, TBusMessageQueue* q); + void Listen(const TVector<TBindResult>& bindTo, TBusMessageQueue* q); + TBusConnection* Accept(SOCKET listen); + + inline ::NActor::TActor<TBusSessionImpl, TStatusTag>* GetStatusActor() { + return this; + } + inline ::NActor::TActor<TBusSessionImpl, TConnectionTag>* GetConnectionsActor() { + return this; + } + + typedef THashMap<TBusSocketAddr, TRemoteConnectionPtr> TAddrRemoteConnections; + + void SendSnapshotToStatusActor(); - void InsertConnectionLockAcquired(TRemoteConnection* connection); - void InsertAcceptorLockAcquired(TAcceptor* acceptor); - - void GetConnections(TVector<TRemoteConnectionPtr>*); - void GetAcceptors(TVector<TAcceptorPtr>*); - void GetConnectionsLockAquired(TVector<TRemoteConnectionPtr>*); - void GetAcceptorsLockAquired(TVector<TAcceptorPtr>*); - - TRemoteConnectionPtr GetConnection(const TBusSocketAddr& addr, bool create); - TRemoteConnectionPtr GetConnectionById(ui64 id); - TAcceptorPtr GetAcceptorById(ui64 id); - - void InvokeOnError(TNonDestroyingAutoPtr<TBusMessage>, EMessageStatus); - - void Cron(); + void InsertConnectionLockAcquired(TRemoteConnection* connection); + void InsertAcceptorLockAcquired(TAcceptor* acceptor); + + void GetConnections(TVector<TRemoteConnectionPtr>*); + void GetAcceptors(TVector<TAcceptorPtr>*); + void GetConnectionsLockAquired(TVector<TRemoteConnectionPtr>*); + void GetAcceptorsLockAquired(TVector<TAcceptorPtr>*); + + TRemoteConnectionPtr GetConnection(const TBusSocketAddr& addr, bool create); + TRemoteConnectionPtr GetConnectionById(ui64 id); + TAcceptorPtr GetAcceptorById(ui64 id); + + void InvokeOnError(TNonDestroyingAutoPtr<TBusMessage>, EMessageStatus); + + void Cron(); - TBusSessionJobCount JobCount; - - // TODO: replace with actor - TMutex ConnectionsLock; + TBusSessionJobCount JobCount; + + // TODO: replace with actor + TMutex ConnectionsLock; - struct TImpl; - THolder<TImpl> Impl; + struct TImpl; + THolder<TImpl> Impl; - const bool IsSource_; + const bool IsSource_; - TBusMessageQueue* const Queue; - TBusProtocol* const Proto; - // copied to be available after Proto dies - const TString ProtoName; + TBusMessageQueue* const Queue; + TBusProtocol* const Proto; + // copied to be available after Proto dies + const TString ProtoName; - IBusErrorHandler* const ErrorHandler; - TUseCountHolder HandlerUseCountHolder; - TBusSessionConfig Config; // TODO: make const + IBusErrorHandler* const ErrorHandler; + TUseCountHolder HandlerUseCountHolder; + TBusSessionConfig Config; // TODO: make const - NEventLoop::TEventLoop WriteEventLoop; - NEventLoop::TEventLoop ReadEventLoop; - THolder<NThreading::TLegacyFuture<void, false>> ReadEventLoopThread; - THolder<NThreading::TLegacyFuture<void, false>> WriteEventLoopThread; + NEventLoop::TEventLoop WriteEventLoop; + NEventLoop::TEventLoop ReadEventLoop; + THolder<NThreading::TLegacyFuture<void, false>> ReadEventLoopThread; + THolder<NThreading::TLegacyFuture<void, false>> WriteEventLoopThread; - THashMap<ui64, TRemoteConnectionPtr> ConnectionsById; - TAddrRemoteConnections Connections; - TAcceptorsPtrs Acceptors; + THashMap<ui64, TRemoteConnectionPtr> ConnectionsById; + TAddrRemoteConnections Connections; + TAcceptorsPtrs Acceptors; - struct TStatusData { - TAtomicSharedPtr<TConnectionsAcceptorsSnapshot> ConnectionsAcceptorsSnapshot; - ::NActor::TQueueForActor<TAtomicSharedPtr<TConnectionsAcceptorsSnapshot>> ConnectionsAcceptorsSnapshotsQueue; + struct TStatusData { + TAtomicSharedPtr<TConnectionsAcceptorsSnapshot> ConnectionsAcceptorsSnapshot; + ::NActor::TQueueForActor<TAtomicSharedPtr<TConnectionsAcceptorsSnapshot>> ConnectionsAcceptorsSnapshotsQueue; - TAtomicShutdownState ShutdownState; + TAtomicShutdownState ShutdownState; - TBusSessionStatus Status; + TBusSessionStatus Status; - TSessionDumpStatus StatusDumpCached; - TMutex StatusDumpCachedMutex; - TInstant StatusDumpCachedLastUpdate; + TSessionDumpStatus StatusDumpCached; + TMutex StatusDumpCachedMutex; + TInstant StatusDumpCachedLastUpdate; - TStatusData(); - }; - TStatusData StatusData; + TStatusData(); + }; + TStatusData StatusData; - struct TConnectionsData { - TAtomicShutdownState ShutdownState; + struct TConnectionsData { + TAtomicShutdownState ShutdownState; - TConnectionsData(); - }; - TConnectionsData ConnectionsData; + TConnectionsData(); + }; + TConnectionsData ConnectionsData; - ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionWriterIncrementalStatus, - TStatusTag, TDeadConnectionTag>* - GetDeadConnectionWriterStatusQueue() { - return this; - } + ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionWriterIncrementalStatus, + TStatusTag, TDeadConnectionTag>* + GetDeadConnectionWriterStatusQueue() { + return this; + } - ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionReaderIncrementalStatus, - TStatusTag, TDeadConnectionTag>* - GetDeadConnectionReaderStatusQueue() { - return this; - } + ::NActor::TQueueInActor<TBusSessionImpl, TRemoteConnectionReaderIncrementalStatus, + TStatusTag, TDeadConnectionTag>* + GetDeadConnectionReaderStatusQueue() { + return this; + } - ::NActor::TQueueInActor<TBusSessionImpl, TAcceptorStatus, - TStatusTag, TDeadConnectionTag>* - GetDeadAcceptorStatusQueue() { - return this; - } + ::NActor::TQueueInActor<TBusSessionImpl, TAcceptorStatus, + TStatusTag, TDeadConnectionTag>* + GetDeadAcceptorStatusQueue() { + return this; + } - template <typename TItem> - ::NActor::IQueueInActor<TItem>* GetQueue() { - return this; - } + template <typename TItem> + ::NActor::IQueueInActor<TItem>* GetQueue() { + return this; + } - ui64 LastAcceptorId; - ui64 LastConnectionId; + ui64 LastAcceptorId; + ui64 LastConnectionId; - TAtomic Down; + TAtomic Down; TSystemEvent ShutdownCompleteEvent; - }; + }; - inline TBusProtocol* TBusSessionImpl::GetProto() const noexcept { - return Proto; - } + inline TBusProtocol* TBusSessionImpl::GetProto() const noexcept { + return Proto; + } - inline const TBusSessionConfig* TBusSessionImpl::GetConfig() const noexcept { - return &Config; - } + inline const TBusSessionConfig* TBusSessionImpl::GetConfig() const noexcept { + return &Config; + } - inline TBusMessageQueue* TBusSessionImpl::GetQueue() const noexcept { - return Queue; - } + inline TBusMessageQueue* TBusSessionImpl::GetQueue() const noexcept { + return Queue; + } } -} +} diff --git a/library/cpp/messagebus/session_job_count.cpp b/library/cpp/messagebus/session_job_count.cpp index 33322b1910..50b27e2f94 100644 --- a/library/cpp/messagebus/session_job_count.cpp +++ b/library/cpp/messagebus/session_job_count.cpp @@ -7,8 +7,8 @@ using namespace NBus::NPrivate; TBusSessionJobCount::TBusSessionJobCount() : JobCount(0) -{ -} +{ +} TBusSessionJobCount::~TBusSessionJobCount() { Y_VERIFY(JobCount == 0, "must be 0 job count to destroy job"); diff --git a/library/cpp/messagebus/session_job_count.h b/library/cpp/messagebus/session_job_count.h index 23aca618b1..ff56c7b392 100644 --- a/library/cpp/messagebus/session_job_count.h +++ b/library/cpp/messagebus/session_job_count.h @@ -4,36 +4,36 @@ #include <util/system/condvar.h> #include <util/system/mutex.h> -namespace NBus { - namespace NPrivate { - class TBusSessionJobCount { - private: - TAtomic JobCount; +namespace NBus { + namespace NPrivate { + class TBusSessionJobCount { + private: + TAtomic JobCount; - TMutex Mutex; - TCondVar CondVar; + TMutex Mutex; + TCondVar CondVar; - public: - TBusSessionJobCount(); - ~TBusSessionJobCount(); + public: + TBusSessionJobCount(); + ~TBusSessionJobCount(); - void Add(unsigned delta) { - AtomicAdd(JobCount, delta); - } + void Add(unsigned delta) { + AtomicAdd(JobCount, delta); + } - void Increment() { - Add(1); - } + void Increment() { + Add(1); + } - void Decrement() { - if (AtomicDecrement(JobCount) == 0) { - TGuard<TMutex> guard(Mutex); - CondVar.BroadCast(); - } - } + void Decrement() { + if (AtomicDecrement(JobCount) == 0) { + TGuard<TMutex> guard(Mutex); + CondVar.BroadCast(); + } + } - void WaitForZero(); - }; + void WaitForZero(); + }; } -} +} diff --git a/library/cpp/messagebus/shutdown_state.cpp b/library/cpp/messagebus/shutdown_state.cpp index a4e2bfa8b2..86ef5d5324 100644 --- a/library/cpp/messagebus/shutdown_state.cpp +++ b/library/cpp/messagebus/shutdown_state.cpp @@ -2,19 +2,19 @@ #include <util/system/yassert.h> -void TAtomicShutdownState::ShutdownCommand() { +void TAtomicShutdownState::ShutdownCommand() { Y_VERIFY(State.CompareAndSet(SS_RUNNING, SS_SHUTDOWN_COMMAND)); } -void TAtomicShutdownState::CompleteShutdown() { +void TAtomicShutdownState::CompleteShutdown() { Y_VERIFY(State.CompareAndSet(SS_SHUTDOWN_COMMAND, SS_SHUTDOWN_COMPLETE)); ShutdownComplete.Signal(); } -bool TAtomicShutdownState::IsRunning() { +bool TAtomicShutdownState::IsRunning() { return State.Get() == SS_RUNNING; } -TAtomicShutdownState::~TAtomicShutdownState() { +TAtomicShutdownState::~TAtomicShutdownState() { Y_VERIFY(SS_SHUTDOWN_COMPLETE == State.Get()); } diff --git a/library/cpp/messagebus/socket_addr.cpp b/library/cpp/messagebus/socket_addr.cpp index c1b3a28fbe..0b4095abf0 100644 --- a/library/cpp/messagebus/socket_addr.cpp +++ b/library/cpp/messagebus/socket_addr.cpp @@ -11,66 +11,66 @@ using namespace NAddr; using namespace NBus; using namespace NBus::NPrivate; -static_assert(ADDR_UNSPEC == 0, "expect ADDR_UNSPEC == 0"); +static_assert(ADDR_UNSPEC == 0, "expect ADDR_UNSPEC == 0"); NBus::NPrivate::TBusSocketAddr::TBusSocketAddr(const NAddr::IRemoteAddr* addr) : IPv6ScopeID(0) { const sockaddr* sa = addr->Addr(); - switch ((EAddrFamily)sa->sa_family) { - case AF_UNSPEC: { - IpAddr.Clear(); - Port = 0; - break; - } - case AF_INET: { - IpAddr.SetInAddr(((const sockaddr_in*)sa)->sin_addr); - Port = InetToHost(((const sockaddr_in*)sa)->sin_port); - break; - } - case AF_INET6: { - IpAddr.SetIn6Addr(((const sockaddr_in6*)sa)->sin6_addr); - Port = InetToHost(((const sockaddr_in*)sa)->sin_port); - IPv6ScopeID = InetToHost(((const sockaddr_in6*)sa)->sin6_scope_id); - break; - } - default: - Y_FAIL("unknown address family"); + switch ((EAddrFamily)sa->sa_family) { + case AF_UNSPEC: { + IpAddr.Clear(); + Port = 0; + break; + } + case AF_INET: { + IpAddr.SetInAddr(((const sockaddr_in*)sa)->sin_addr); + Port = InetToHost(((const sockaddr_in*)sa)->sin_port); + break; + } + case AF_INET6: { + IpAddr.SetIn6Addr(((const sockaddr_in6*)sa)->sin6_addr); + Port = InetToHost(((const sockaddr_in*)sa)->sin_port); + IPv6ScopeID = InetToHost(((const sockaddr_in6*)sa)->sin6_scope_id); + break; + } + default: + Y_FAIL("unknown address family"); } } -NBus::NPrivate::TBusSocketAddr::TBusSocketAddr(TStringBuf host, unsigned port) { +NBus::NPrivate::TBusSocketAddr::TBusSocketAddr(TStringBuf host, unsigned port) { *this = TNetAddr(host, port); } -NBus::NPrivate::TBusSocketAddr::TBusSocketAddr(const TNetAddr& addr) { +NBus::NPrivate::TBusSocketAddr::TBusSocketAddr(const TNetAddr& addr) { *this = TBusSocketAddr(&addr); } -TNetAddr NBus::NPrivate::TBusSocketAddr::ToNetAddr() const { +TNetAddr NBus::NPrivate::TBusSocketAddr::ToNetAddr() const { sockaddr_storage storage; Zero(storage); - storage.ss_family = (ui16)IpAddr.GetAddrFamily(); + storage.ss_family = (ui16)IpAddr.GetAddrFamily(); switch (IpAddr.GetAddrFamily()) { - case ADDR_UNSPEC: - return TNetAddr(); - case ADDR_IPV4: { - ((sockaddr_in*)&storage)->sin_addr = IpAddr.GetInAddr(); - ((sockaddr_in*)&storage)->sin_port = HostToInet(Port); - break; - } - case ADDR_IPV6: { - ((sockaddr_in6*)&storage)->sin6_addr = IpAddr.GetIn6Addr(); - ((sockaddr_in6*)&storage)->sin6_port = HostToInet(Port); - ((sockaddr_in6*)&storage)->sin6_scope_id = HostToInet(IPv6ScopeID); - break; - } + case ADDR_UNSPEC: + return TNetAddr(); + case ADDR_IPV4: { + ((sockaddr_in*)&storage)->sin_addr = IpAddr.GetInAddr(); + ((sockaddr_in*)&storage)->sin_port = HostToInet(Port); + break; + } + case ADDR_IPV6: { + ((sockaddr_in6*)&storage)->sin6_addr = IpAddr.GetIn6Addr(); + ((sockaddr_in6*)&storage)->sin6_port = HostToInet(Port); + ((sockaddr_in6*)&storage)->sin6_scope_id = HostToInet(IPv6ScopeID); + break; + } } - return TNetAddr(new TOpaqueAddr((sockaddr*)&storage)); + return TNetAddr(new TOpaqueAddr((sockaddr*)&storage)); } template <> diff --git a/library/cpp/messagebus/socket_addr.h b/library/cpp/messagebus/socket_addr.h index 959eafe689..e66735c458 100644 --- a/library/cpp/messagebus/socket_addr.h +++ b/library/cpp/messagebus/socket_addr.h @@ -13,90 +13,90 @@ namespace NBus { class TNetAddr; } -namespace NBus { - namespace NPrivate { - enum EAddrFamily { - ADDR_UNSPEC = AF_UNSPEC, - ADDR_IPV4 = AF_INET, - ADDR_IPV6 = AF_INET6, - }; - - class TBusIpAddr { - private: - EAddrFamily Af; - - union { - in_addr In4; - in6_addr In6; - }; - - public: - TBusIpAddr() { - Clear(); - } - - EAddrFamily GetAddrFamily() const { - return Af; - } - - void Clear() { - Zero(*this); - } - - in_addr GetInAddr() const { - Y_ASSERT(Af == ADDR_IPV4); - return In4; - } - - void SetInAddr(const in_addr& in4) { - Clear(); - Af = ADDR_IPV4; - In4 = in4; - } - - in6_addr GetIn6Addr() const { - Y_ASSERT(Af == ADDR_IPV6); - return In6; - } - - void SetIn6Addr(const in6_addr& in6) { - Clear(); - Af = ADDR_IPV6; - In6 = in6; - } - - bool operator==(const TBusIpAddr& that) const { - return memcmp(this, &that, sizeof(that)) == 0; - } - }; - - class TBusSocketAddr { - public: - TBusIpAddr IpAddr; - ui16 Port; - - //Only makes sense for IPv6 link-local addresses - ui32 IPv6ScopeID; - - TBusSocketAddr() - : Port(0) - , IPv6ScopeID(0) - { - } - - TBusSocketAddr(const NAddr::IRemoteAddr*); - TBusSocketAddr(const TNetAddr&); - TBusSocketAddr(TStringBuf host, unsigned port); - - TNetAddr ToNetAddr() const; - - bool operator==(const TBusSocketAddr& that) const { - return IpAddr == that.IpAddr && Port == that.Port; - } - }; - - } -} +namespace NBus { + namespace NPrivate { + enum EAddrFamily { + ADDR_UNSPEC = AF_UNSPEC, + ADDR_IPV4 = AF_INET, + ADDR_IPV6 = AF_INET6, + }; + + class TBusIpAddr { + private: + EAddrFamily Af; + + union { + in_addr In4; + in6_addr In6; + }; + + public: + TBusIpAddr() { + Clear(); + } + + EAddrFamily GetAddrFamily() const { + return Af; + } + + void Clear() { + Zero(*this); + } + + in_addr GetInAddr() const { + Y_ASSERT(Af == ADDR_IPV4); + return In4; + } + + void SetInAddr(const in_addr& in4) { + Clear(); + Af = ADDR_IPV4; + In4 = in4; + } + + in6_addr GetIn6Addr() const { + Y_ASSERT(Af == ADDR_IPV6); + return In6; + } + + void SetIn6Addr(const in6_addr& in6) { + Clear(); + Af = ADDR_IPV6; + In6 = in6; + } + + bool operator==(const TBusIpAddr& that) const { + return memcmp(this, &that, sizeof(that)) == 0; + } + }; + + class TBusSocketAddr { + public: + TBusIpAddr IpAddr; + ui16 Port; + + //Only makes sense for IPv6 link-local addresses + ui32 IPv6ScopeID; + + TBusSocketAddr() + : Port(0) + , IPv6ScopeID(0) + { + } + + TBusSocketAddr(const NAddr::IRemoteAddr*); + TBusSocketAddr(const TNetAddr&); + TBusSocketAddr(TStringBuf host, unsigned port); + + TNetAddr ToNetAddr() const; + + bool operator==(const TBusSocketAddr& that) const { + return IpAddr == that.IpAddr && Port == that.Port; + } + }; + + } +} template <> struct THash<NBus::NPrivate::TBusIpAddr> { diff --git a/library/cpp/messagebus/storage.cpp b/library/cpp/messagebus/storage.cpp index efefc87340..030284a74c 100644 --- a/library/cpp/messagebus/storage.cpp +++ b/library/cpp/messagebus/storage.cpp @@ -2,159 +2,159 @@ #include <typeinfo> -namespace NBus { - namespace NPrivate { - TTimedMessages::TTimedMessages() { - } - - TTimedMessages::~TTimedMessages() { - Y_VERIFY(Items.empty()); - } - - void TTimedMessages::PushBack(TNonDestroyingAutoPtr<TBusMessage> m) { - TItem i; - i.Message.Reset(m.Release()); - Items.push_back(i); - } - - TNonDestroyingAutoPtr<TBusMessage> TTimedMessages::PopFront() { - TBusMessage* r = nullptr; - if (!Items.empty()) { - r = Items.front()->Message.Release(); - Items.pop_front(); - } - return r; - } - - bool TTimedMessages::Empty() const { - return Items.empty(); - } - - size_t TTimedMessages::Size() const { - return Items.size(); - } - - void TTimedMessages::Timeout(TInstant before, TMessagesPtrs* r) { - // shortcut - if (before == TInstant::Max()) { - Clear(r); - return; - } - - while (!Items.empty()) { - TItem& i = *Items.front(); - if (TInstant::MilliSeconds(i.Message->GetHeader()->SendTime) > before) { - break; - } - r->push_back(i.Message.Release()); - Items.pop_front(); - } - } - - void TTimedMessages::Clear(TMessagesPtrs* r) { - while (!Items.empty()) { - r->push_back(Items.front()->Message.Release()); - Items.pop_front(); - } - } - - TSyncAckMessages::TSyncAckMessages() { - KeyToMessage.set_empty_key(0); - KeyToMessage.set_deleted_key(1); - } - - TSyncAckMessages::~TSyncAckMessages() { - Y_VERIFY(KeyToMessage.empty()); - Y_VERIFY(TimedItems.empty()); - } - - void TSyncAckMessages::Push(TBusMessagePtrAndHeader& m) { - // Perform garbage collection if `TimedMessages` contain too many junk data - if (TimedItems.size() > 1000 && TimedItems.size() > KeyToMessage.size() * 4) { - Gc(); - } - - TValue value = {m.MessagePtr.Release()}; - - std::pair<TKeyToMessage::iterator, bool> p = KeyToMessage.insert(TKeyToMessage::value_type(m.Header.Id, value)); +namespace NBus { + namespace NPrivate { + TTimedMessages::TTimedMessages() { + } + + TTimedMessages::~TTimedMessages() { + Y_VERIFY(Items.empty()); + } + + void TTimedMessages::PushBack(TNonDestroyingAutoPtr<TBusMessage> m) { + TItem i; + i.Message.Reset(m.Release()); + Items.push_back(i); + } + + TNonDestroyingAutoPtr<TBusMessage> TTimedMessages::PopFront() { + TBusMessage* r = nullptr; + if (!Items.empty()) { + r = Items.front()->Message.Release(); + Items.pop_front(); + } + return r; + } + + bool TTimedMessages::Empty() const { + return Items.empty(); + } + + size_t TTimedMessages::Size() const { + return Items.size(); + } + + void TTimedMessages::Timeout(TInstant before, TMessagesPtrs* r) { + // shortcut + if (before == TInstant::Max()) { + Clear(r); + return; + } + + while (!Items.empty()) { + TItem& i = *Items.front(); + if (TInstant::MilliSeconds(i.Message->GetHeader()->SendTime) > before) { + break; + } + r->push_back(i.Message.Release()); + Items.pop_front(); + } + } + + void TTimedMessages::Clear(TMessagesPtrs* r) { + while (!Items.empty()) { + r->push_back(Items.front()->Message.Release()); + Items.pop_front(); + } + } + + TSyncAckMessages::TSyncAckMessages() { + KeyToMessage.set_empty_key(0); + KeyToMessage.set_deleted_key(1); + } + + TSyncAckMessages::~TSyncAckMessages() { + Y_VERIFY(KeyToMessage.empty()); + Y_VERIFY(TimedItems.empty()); + } + + void TSyncAckMessages::Push(TBusMessagePtrAndHeader& m) { + // Perform garbage collection if `TimedMessages` contain too many junk data + if (TimedItems.size() > 1000 && TimedItems.size() > KeyToMessage.size() * 4) { + Gc(); + } + + TValue value = {m.MessagePtr.Release()}; + + std::pair<TKeyToMessage::iterator, bool> p = KeyToMessage.insert(TKeyToMessage::value_type(m.Header.Id, value)); Y_VERIFY(p.second, "non-unique id; %s", value.Message->Describe().data()); - TTimedItem item = {m.Header.Id, m.Header.SendTime}; - TimedItems.push_back(item); - } + TTimedItem item = {m.Header.Id, m.Header.SendTime}; + TimedItems.push_back(item); + } - TBusMessage* TSyncAckMessages::Pop(TBusKey id) { - TKeyToMessage::iterator it = KeyToMessage.find(id); - if (it == KeyToMessage.end()) { - return nullptr; - } - TValue v = it->second; - KeyToMessage.erase(it); + TBusMessage* TSyncAckMessages::Pop(TBusKey id) { + TKeyToMessage::iterator it = KeyToMessage.find(id); + if (it == KeyToMessage.end()) { + return nullptr; + } + TValue v = it->second; + KeyToMessage.erase(it); - // `TimedMessages` still contain record about this message + // `TimedMessages` still contain record about this message - return v.Message; - } + return v.Message; + } - void TSyncAckMessages::Timeout(TInstant before, TMessagesPtrs* r) { - // shortcut - if (before == TInstant::Max()) { - Clear(r); - return; - } + void TSyncAckMessages::Timeout(TInstant before, TMessagesPtrs* r) { + // shortcut + if (before == TInstant::Max()) { + Clear(r); + return; + } - Y_ASSERT(r->empty()); + Y_ASSERT(r->empty()); - while (!TimedItems.empty()) { - TTimedItem i = TimedItems.front(); - if (TInstant::MilliSeconds(i.SendTime) > before) { - break; - } + while (!TimedItems.empty()) { + TTimedItem i = TimedItems.front(); + if (TInstant::MilliSeconds(i.SendTime) > before) { + break; + } - TKeyToMessage::iterator itMessage = KeyToMessage.find(i.Key); + TKeyToMessage::iterator itMessage = KeyToMessage.find(i.Key); - if (itMessage != KeyToMessage.end()) { - r->push_back(itMessage->second.Message); - KeyToMessage.erase(itMessage); - } + if (itMessage != KeyToMessage.end()) { + r->push_back(itMessage->second.Message); + KeyToMessage.erase(itMessage); + } - TimedItems.pop_front(); - } + TimedItems.pop_front(); + } } - void TSyncAckMessages::Clear(TMessagesPtrs* r) { - for (TKeyToMessage::const_iterator i = KeyToMessage.begin(); i != KeyToMessage.end(); ++i) { - r->push_back(i->second.Message); - } + void TSyncAckMessages::Clear(TMessagesPtrs* r) { + for (TKeyToMessage::const_iterator i = KeyToMessage.begin(); i != KeyToMessage.end(); ++i) { + r->push_back(i->second.Message); + } - KeyToMessage.clear(); - TimedItems.clear(); + KeyToMessage.clear(); + TimedItems.clear(); } - void TSyncAckMessages::Gc() { - TDeque<TTimedItem> tmp; + void TSyncAckMessages::Gc() { + TDeque<TTimedItem> tmp; - for (auto& timedItem : TimedItems) { - if (KeyToMessage.find(timedItem.Key) == KeyToMessage.end()) { - continue; - } - tmp.push_back(timedItem); - } + for (auto& timedItem : TimedItems) { + if (KeyToMessage.find(timedItem.Key) == KeyToMessage.end()) { + continue; + } + tmp.push_back(timedItem); + } - TimedItems.swap(tmp); - } + TimedItems.swap(tmp); + } - void TSyncAckMessages::RemoveAll(const TMessagesPtrs& messages) { - for (auto message : messages) { - TKeyToMessage::iterator it = KeyToMessage.find(message->GetHeader()->Id); - Y_VERIFY(it != KeyToMessage.end(), "delete non-existent message"); - KeyToMessage.erase(it); - } - } + void TSyncAckMessages::RemoveAll(const TMessagesPtrs& messages) { + for (auto message : messages) { + TKeyToMessage::iterator it = KeyToMessage.find(message->GetHeader()->Id); + Y_VERIFY(it != KeyToMessage.end(), "delete non-existent message"); + KeyToMessage.erase(it); + } + } - void TSyncAckMessages::DumpState() { - Cerr << TimedItems.size() << Endl; - Cerr << KeyToMessage.size() << Endl; + void TSyncAckMessages::DumpState() { + Cerr << TimedItems.size() << Endl; + Cerr << KeyToMessage.size() << Endl; } } diff --git a/library/cpp/messagebus/storage.h b/library/cpp/messagebus/storage.h index 7d168844ed..694ea2fcfe 100644 --- a/library/cpp/messagebus/storage.h +++ b/library/cpp/messagebus/storage.h @@ -10,85 +10,85 @@ #include <util/generic/noncopyable.h> #include <util/generic/utility.h> -namespace NBus { - namespace NPrivate { - typedef TVector<TBusMessage*> TMessagesPtrs; +namespace NBus { + namespace NPrivate { + typedef TVector<TBusMessage*> TMessagesPtrs; - class TTimedMessages { - public: - TTimedMessages(); - ~TTimedMessages(); + class TTimedMessages { + public: + TTimedMessages(); + ~TTimedMessages(); - struct TItem { - THolder<TBusMessage> Message; + struct TItem { + THolder<TBusMessage> Message; - void Swap(TItem& that) { - DoSwap(Message, that.Message); - } - }; + void Swap(TItem& that) { + DoSwap(Message, that.Message); + } + }; - typedef TDeque<TMoved<TItem>> TItems; + typedef TDeque<TMoved<TItem>> TItems; - void PushBack(TNonDestroyingAutoPtr<TBusMessage> m); - TNonDestroyingAutoPtr<TBusMessage> PopFront(); - bool Empty() const; - size_t Size() const; + void PushBack(TNonDestroyingAutoPtr<TBusMessage> m); + TNonDestroyingAutoPtr<TBusMessage> PopFront(); + bool Empty() const; + size_t Size() const; - void Timeout(TInstant before, TMessagesPtrs* r); - void Clear(TMessagesPtrs* r); + void Timeout(TInstant before, TMessagesPtrs* r); + void Clear(TMessagesPtrs* r); - private: - TItems Items; - }; + private: + TItems Items; + }; - class TSyncAckMessages : TNonCopyable { - public: - TSyncAckMessages(); - ~TSyncAckMessages(); + class TSyncAckMessages : TNonCopyable { + public: + TSyncAckMessages(); + ~TSyncAckMessages(); - void Push(TBusMessagePtrAndHeader& m); - TBusMessage* Pop(TBusKey id); + void Push(TBusMessagePtrAndHeader& m); + TBusMessage* Pop(TBusKey id); - void Timeout(TInstant before, TMessagesPtrs* r); + void Timeout(TInstant before, TMessagesPtrs* r); - void Clear(TMessagesPtrs* r); + void Clear(TMessagesPtrs* r); - size_t Size() const { - return KeyToMessage.size(); - } + size_t Size() const { + return KeyToMessage.size(); + } - void RemoveAll(const TMessagesPtrs&); + void RemoveAll(const TMessagesPtrs&); - void Gc(); + void Gc(); - void DumpState(); + void DumpState(); - private: - struct TTimedItem { - TBusKey Key; - TBusInstant SendTime; - }; + private: + struct TTimedItem { + TBusKey Key; + TBusInstant SendTime; + }; - typedef TDeque<TTimedItem> TTimedItems; - typedef TDeque<TTimedItem>::iterator TTimedIterator; + typedef TDeque<TTimedItem> TTimedItems; + typedef TDeque<TTimedItem>::iterator TTimedIterator; - TTimedItems TimedItems; + TTimedItems TimedItems; - struct TValue { - TBusMessage* Message; - }; + struct TValue { + TBusMessage* Message; + }; - // keys are already random, no need to hash them further - struct TIdHash { - size_t operator()(TBusKey value) const { - return value; - } - }; + // keys are already random, no need to hash them further + struct TIdHash { + size_t operator()(TBusKey value) const { + return value; + } + }; typedef google::dense_hash_map<TBusKey, TValue, TIdHash> TKeyToMessage; - TKeyToMessage KeyToMessage; + TKeyToMessage KeyToMessage; }; - } -} + } +} diff --git a/library/cpp/messagebus/synchandler.cpp b/library/cpp/messagebus/synchandler.cpp index 8e891d66b3..3498efbb21 100644 --- a/library/cpp/messagebus/synchandler.cpp +++ b/library/cpp/messagebus/synchandler.cpp @@ -13,43 +13,43 @@ using namespace NBus::NPrivate; /// 3. Message reply /// 4. Reply status struct TBusSyncMessageData { - TCondVar ReplyEvent; - TMutex ReplyLock; - TBusMessage* Reply; + TCondVar ReplyEvent; + TMutex ReplyLock; + TBusMessage* Reply; EMessageStatus ReplyStatus; TBusSyncMessageData() : Reply(nullptr) , ReplyStatus(MESSAGE_DONT_ASK) - { - } + { + } }; -class TSyncHandler: public IBusClientHandler { +class TSyncHandler: public IBusClientHandler { public: TSyncHandler(bool expectReply = true) : ExpectReply(expectReply) , Session(nullptr) - { - } - ~TSyncHandler() override { - } + { + } + ~TSyncHandler() override { + } void OnReply(TAutoPtr<TBusMessage> pMessage0, TAutoPtr<TBusMessage> pReply0) override { TBusMessage* pMessage = pMessage0.Release(); TBusMessage* pReply = pReply0.Release(); - if (!ExpectReply) { // Maybe need VERIFY, but it will be better to support backward compatibility here. + if (!ExpectReply) { // Maybe need VERIFY, but it will be better to support backward compatibility here. return; } - TBusSyncMessageData* data = static_cast<TBusSyncMessageData*>(pMessage->Data); + TBusSyncMessageData* data = static_cast<TBusSyncMessageData*>(pMessage->Data); SignalResult(data, pReply, MESSAGE_OK); } void OnError(TAutoPtr<TBusMessage> pMessage0, EMessageStatus status) override { TBusMessage* pMessage = pMessage0.Release(); - TBusSyncMessageData* data = static_cast<TBusSyncMessageData*>(pMessage->Data); + TBusSyncMessageData* data = static_cast<TBusSyncMessageData*>(pMessage->Data); if (!data) { return; } @@ -64,7 +64,7 @@ public: void OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage) override { Y_ASSERT(!ExpectReply); - TBusSyncMessageData* data = static_cast<TBusSyncMessageData*>(pMessage.Release()->Data); + TBusSyncMessageData* data = static_cast<TBusSyncMessageData*>(pMessage.Release()->Data); SignalResult(data, /*pReply=*/nullptr, MESSAGE_OK); } @@ -75,7 +75,7 @@ public: } private: - void SignalResult(TBusSyncMessageData* data, TBusMessage* pReply, EMessageStatus status) const { + void SignalResult(TBusSyncMessageData* data, TBusMessage* pReply, EMessageStatus status) const { Y_VERIFY(data, "Message data is set to NULL."); TGuard<TMutex> G(data->ReplyLock); data->Reply = pReply; @@ -89,76 +89,76 @@ private: TRemoteClientSession* Session; }; -namespace NBus { - namespace NPrivate { +namespace NBus { + namespace NPrivate { #ifdef _MSC_VER #pragma warning(push) -#pragma warning(disable : 4250) // 'NBus::NPrivate::TRemoteClientSession' : inherits 'NBus::NPrivate::TBusSessionImpl::NBus::NPrivate::TBusSessionImpl::GetConfig' via dominance +#pragma warning(disable : 4250) // 'NBus::NPrivate::TRemoteClientSession' : inherits 'NBus::NPrivate::TBusSessionImpl::NBus::NPrivate::TBusSessionImpl::GetConfig' via dominance #endif - /////////////////////////////////////////////////////////////////////////// - class TBusSyncSourceSessionImpl - : private TSyncHandler - // TODO: do not extend TRemoteClientSession - , - public TRemoteClientSession { - private: - bool NeedReply; - - public: - TBusSyncSourceSessionImpl(TBusMessageQueue* queue, TBusProtocol* proto, const TBusClientSessionConfig& config, bool needReply, const TString& name) - : TSyncHandler(needReply) - , TRemoteClientSession(queue, proto, this, config, name) - , NeedReply(needReply) - { - SetSession(this); - } - - TBusMessage* SendSyncMessage(TBusMessage* pMessage, EMessageStatus& status, const TNetAddr* addr = nullptr) { - Y_VERIFY(!Queue->GetExecutor()->IsInExecutorThread(), - "SendSyncMessage must not be called from executor thread"); - - TBusMessage* reply = nullptr; - THolder<TBusSyncMessageData> data(new TBusSyncMessageData()); - - pMessage->Data = data.Get(); - - { - TGuard<TMutex> G(data->ReplyLock); - if (NeedReply) { - status = SendMessage(pMessage, addr, false); // probably should be true - } else { - status = SendMessageOneWay(pMessage, addr); - } - - if (status == MESSAGE_OK) { - data->ReplyEvent.Wait(data->ReplyLock); - TBusSyncMessageData* rdata = static_cast<TBusSyncMessageData*>(pMessage->Data); - Y_VERIFY(rdata == data.Get(), "Message data pointer should not be modified."); - reply = rdata->Reply; - status = rdata->ReplyStatus; - } - } - - // deletion of message and reply is a job of application. - pMessage->Data = nullptr; - - return reply; - } - }; - + /////////////////////////////////////////////////////////////////////////// + class TBusSyncSourceSessionImpl + : private TSyncHandler + // TODO: do not extend TRemoteClientSession + , + public TRemoteClientSession { + private: + bool NeedReply; + + public: + TBusSyncSourceSessionImpl(TBusMessageQueue* queue, TBusProtocol* proto, const TBusClientSessionConfig& config, bool needReply, const TString& name) + : TSyncHandler(needReply) + , TRemoteClientSession(queue, proto, this, config, name) + , NeedReply(needReply) + { + SetSession(this); + } + + TBusMessage* SendSyncMessage(TBusMessage* pMessage, EMessageStatus& status, const TNetAddr* addr = nullptr) { + Y_VERIFY(!Queue->GetExecutor()->IsInExecutorThread(), + "SendSyncMessage must not be called from executor thread"); + + TBusMessage* reply = nullptr; + THolder<TBusSyncMessageData> data(new TBusSyncMessageData()); + + pMessage->Data = data.Get(); + + { + TGuard<TMutex> G(data->ReplyLock); + if (NeedReply) { + status = SendMessage(pMessage, addr, false); // probably should be true + } else { + status = SendMessageOneWay(pMessage, addr); + } + + if (status == MESSAGE_OK) { + data->ReplyEvent.Wait(data->ReplyLock); + TBusSyncMessageData* rdata = static_cast<TBusSyncMessageData*>(pMessage->Data); + Y_VERIFY(rdata == data.Get(), "Message data pointer should not be modified."); + reply = rdata->Reply; + status = rdata->ReplyStatus; + } + } + + // deletion of message and reply is a job of application. + pMessage->Data = nullptr; + + return reply; + } + }; + #ifdef _MSC_VER #pragma warning(pop) #endif - } -} + } +} TBusSyncSourceSession::TBusSyncSourceSession(TIntrusivePtr< ::NBus::NPrivate::TBusSyncSourceSessionImpl> session) : Session(session) -{ -} +{ +} -TBusSyncSourceSession::~TBusSyncSourceSession() { +TBusSyncSourceSession::~TBusSyncSourceSession() { Shutdown(); } @@ -170,7 +170,7 @@ TBusMessage* TBusSyncSourceSession::SendSyncMessage(TBusMessage* pMessage, EMess return Session->SendSyncMessage(pMessage, status, addr); } -int TBusSyncSourceSession::RegisterService(const char* hostname, TBusKey start, TBusKey end, EIpVersion ipVersion) { +int TBusSyncSourceSession::RegisterService(const char* hostname, TBusKey start, TBusKey end, EIpVersion ipVersion) { return Session->RegisterService(hostname, start, end, ipVersion); } diff --git a/library/cpp/messagebus/test/example/client/client.cpp b/library/cpp/messagebus/test/example/client/client.cpp index 89b5f2c9be..0a4097f5f4 100644 --- a/library/cpp/messagebus/test/example/client/client.cpp +++ b/library/cpp/messagebus/test/example/client/client.cpp @@ -6,7 +6,7 @@ using namespace NBus; using namespace NCalculator; namespace NCalculator { - struct TCalculatorClient: public IBusClientHandler { + struct TCalculatorClient: public IBusClientHandler { TCalculatorProtocol Proto; TBusMessageQueuePtr MessageQueue; TBusClientSessionPtr ClientSession; diff --git a/library/cpp/messagebus/test/example/common/proto.h b/library/cpp/messagebus/test/example/common/proto.h index a151aac468..904dbad713 100644 --- a/library/cpp/messagebus/test/example/common/proto.h +++ b/library/cpp/messagebus/test/example/common/proto.h @@ -10,7 +10,7 @@ namespace NCalculator { typedef ::NBus::TBusBufferMessage<TRequestMulRecord, 2> TRequestMul; typedef ::NBus::TBusBufferMessage<TResponseRecord, 3> TResponse; - struct TCalculatorProtocol: public ::NBus::TBusBufferProtocol { + struct TCalculatorProtocol: public ::NBus::TBusBufferProtocol { TCalculatorProtocol(); }; diff --git a/library/cpp/messagebus/test/example/server/server.cpp b/library/cpp/messagebus/test/example/server/server.cpp index 13e52d75f5..1d065c1ef6 100644 --- a/library/cpp/messagebus/test/example/server/server.cpp +++ b/library/cpp/messagebus/test/example/server/server.cpp @@ -4,7 +4,7 @@ using namespace NBus; using namespace NCalculator; namespace NCalculator { - struct TCalculatorServer: public IBusServerHandler { + struct TCalculatorServer: public IBusServerHandler { TCalculatorProtocol Proto; TBusMessageQueuePtr MessageQueue; TBusServerSessionPtr ServerSession; @@ -43,7 +43,7 @@ namespace NCalculator { } } }; -} +} int main(int, char**) { TCalculatorServer server; diff --git a/library/cpp/messagebus/test/helper/alloc_counter.h b/library/cpp/messagebus/test/helper/alloc_counter.h index ec9041cb15..7011b61b9d 100644 --- a/library/cpp/messagebus/test/helper/alloc_counter.h +++ b/library/cpp/messagebus/test/helper/alloc_counter.h @@ -4,14 +4,14 @@ #include <util/system/atomic.h> #include <util/system/yassert.h> -class TAllocCounter : TNonCopyable { +class TAllocCounter : TNonCopyable { private: TAtomic* CountPtr; - + public: - TAllocCounter(TAtomic* countPtr) - : CountPtr(countPtr) - { + TAllocCounter(TAtomic* countPtr) + : CountPtr(countPtr) + { AtomicIncrement(*CountPtr); } diff --git a/library/cpp/messagebus/test/helper/example.cpp b/library/cpp/messagebus/test/helper/example.cpp index 7c6d704042..6260541e1b 100644 --- a/library/cpp/messagebus/test/helper/example.cpp +++ b/library/cpp/messagebus/test/helper/example.cpp @@ -9,9 +9,9 @@ using namespace NBus::NTest; static void FillWithJunk(TArrayRef<char> data) { TStringBuf junk = - "01234567890123456789012345678901234567890123456789012345678901234567890123456789" - "01234567890123456789012345678901234567890123456789012345678901234567890123456789" - "01234567890123456789012345678901234567890123456789012345678901234567890123456789" + "01234567890123456789012345678901234567890123456789012345678901234567890123456789" + "01234567890123456789012345678901234567890123456789012345678901234567890123456789" + "01234567890123456789012345678901234567890123456789012345678901234567890123456789" "01234567890123456789012345678901234567890123456789012345678901234567890123456789"; for (size_t i = 0; i < data.size(); i += junk.size()) { @@ -37,8 +37,8 @@ TExampleRequest::TExampleRequest(TAtomic* counterPtr, size_t payloadSize) TExampleRequest::TExampleRequest(ECreateUninitialized, TAtomic* counterPtr) : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) , AllocCounter(counterPtr) -{ -} +{ +} TExampleResponse::TExampleResponse(TAtomic* counterPtr, size_t payloadSize) : TBusMessage(79) @@ -50,8 +50,8 @@ TExampleResponse::TExampleResponse(TAtomic* counterPtr, size_t payloadSize) TExampleResponse::TExampleResponse(ECreateUninitialized, TAtomic* counterPtr) : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) , AllocCounter(counterPtr) -{ -} +{ +} TExampleProtocol::TExampleProtocol(int port) : TBusProtocol("Example", port) @@ -60,8 +60,8 @@ TExampleProtocol::TExampleProtocol(int port) , RequestCountDeserialized(0) , ResponseCountDeserialized(0) , StartCount(0) -{ -} +{ +} TExampleProtocol::~TExampleProtocol() { if (UncaughtException()) { @@ -124,13 +124,13 @@ TExampleClient::TExampleClient(const TBusClientSessionConfig sessionConfig, int TExampleClient::~TExampleClient() { } -EMessageStatus TExampleClient::SendMessage(const TNetAddr* addr) { +EMessageStatus TExampleClient::SendMessage(const TNetAddr* addr) { TAutoPtr<TExampleRequest> message(new TExampleRequest(&Proto.RequestCount, DataSize)); message->SetCompressed(UseCompression); return Session->SendMessageAutoPtr(message, addr); } -void TExampleClient::SendMessages(size_t count, const TNetAddr* addr) { +void TExampleClient::SendMessages(size_t count, const TNetAddr* addr) { UNIT_ASSERT(MessageCount == 0); UNIT_ASSERT(RepliesCount == 0); UNIT_ASSERT(Errors == 0); @@ -184,7 +184,7 @@ void TExampleClient::WaitForError(EMessageStatus status) { UNIT_ASSERT_VALUES_EQUAL(status, error); } -void TExampleClient::SendMessagesWaitReplies(size_t count, const TNetAddr* addr) { +void TExampleClient::SendMessagesWaitReplies(size_t count, const TNetAddr* addr) { SendMessages(count, addr); WaitReplies(); } @@ -215,8 +215,8 @@ void TExampleClient::OnError(TAutoPtr<TBusMessage> mess, EMessageStatus status) } TExampleServer::TExampleServer( - const char* name, - const TBusServerSessionConfig& sessionConfig) + const char* name, + const TBusServerSessionConfig& sessionConfig) : UseCompression(false) , AckMessageBeforeSendReply(false) , ForgetRequest(false) diff --git a/library/cpp/messagebus/test/helper/example.h b/library/cpp/messagebus/test/helper/example.h index 26b7475308..819562719d 100644 --- a/library/cpp/messagebus/test/helper/example.h +++ b/library/cpp/messagebus/test/helper/example.h @@ -10,123 +10,123 @@ #include <util/system/event.h> -namespace NBus { - namespace NTest { - class TExampleRequest: public TBusMessage { - friend class TExampleProtocol; - - private: - TAllocCounter AllocCounter; - - public: - TString Data; - - public: - TExampleRequest(TAtomic* counterPtr, size_t payloadSize = 320); - TExampleRequest(ECreateUninitialized, TAtomic* counterPtr); - }; - - class TExampleResponse: public TBusMessage { - friend class TExampleProtocol; - - private: - TAllocCounter AllocCounter; - - public: - TString Data; - TExampleResponse(TAtomic* counterPtr, size_t payloadSize = 320); - TExampleResponse(ECreateUninitialized, TAtomic* counterPtr); - }; - - class TExampleProtocol: public TBusProtocol { - public: - TAtomic RequestCount; - TAtomic ResponseCount; - TAtomic RequestCountDeserialized; - TAtomic ResponseCountDeserialized; - TAtomic StartCount; - - TExampleProtocol(int port = 0); - - ~TExampleProtocol() override; - - void Serialize(const TBusMessage* message, TBuffer& buffer) override; - - TAutoPtr<TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) override; - }; - - class TExampleClient: private TBusClientHandlerError { - public: - TExampleProtocol Proto; - bool UseCompression; - bool CrashOnError; - size_t DataSize; - - ssize_t MessageCount; - TAtomic RepliesCount; - TAtomic Errors; - EMessageStatus LastError; +namespace NBus { + namespace NTest { + class TExampleRequest: public TBusMessage { + friend class TExampleProtocol; + + private: + TAllocCounter AllocCounter; + + public: + TString Data; + + public: + TExampleRequest(TAtomic* counterPtr, size_t payloadSize = 320); + TExampleRequest(ECreateUninitialized, TAtomic* counterPtr); + }; + + class TExampleResponse: public TBusMessage { + friend class TExampleProtocol; + + private: + TAllocCounter AllocCounter; + + public: + TString Data; + TExampleResponse(TAtomic* counterPtr, size_t payloadSize = 320); + TExampleResponse(ECreateUninitialized, TAtomic* counterPtr); + }; + + class TExampleProtocol: public TBusProtocol { + public: + TAtomic RequestCount; + TAtomic ResponseCount; + TAtomic RequestCountDeserialized; + TAtomic ResponseCountDeserialized; + TAtomic StartCount; + + TExampleProtocol(int port = 0); + + ~TExampleProtocol() override; + + void Serialize(const TBusMessage* message, TBuffer& buffer) override; + + TAutoPtr<TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) override; + }; + + class TExampleClient: private TBusClientHandlerError { + public: + TExampleProtocol Proto; + bool UseCompression; + bool CrashOnError; + size_t DataSize; + + ssize_t MessageCount; + TAtomic RepliesCount; + TAtomic Errors; + EMessageStatus LastError; TSystemEvent WorkDone; - TBusMessageQueuePtr Bus; - TBusClientSessionPtr Session; + TBusMessageQueuePtr Bus; + TBusClientSessionPtr Session; - public: - TExampleClient(const TBusClientSessionConfig sessionConfig = TBusClientSessionConfig(), int port = 0); - ~TExampleClient() override; + public: + TExampleClient(const TBusClientSessionConfig sessionConfig = TBusClientSessionConfig(), int port = 0); + ~TExampleClient() override; - EMessageStatus SendMessage(const TNetAddr* addr = nullptr); + EMessageStatus SendMessage(const TNetAddr* addr = nullptr); - void SendMessages(size_t count, const TNetAddr* addr = nullptr); - void SendMessages(size_t count, const TNetAddr& addr); + void SendMessages(size_t count, const TNetAddr* addr = nullptr); + void SendMessages(size_t count, const TNetAddr& addr); - void ResetCounters(); - void WaitReplies(); - EMessageStatus WaitForError(); - void WaitForError(EMessageStatus status); + void ResetCounters(); + void WaitReplies(); + EMessageStatus WaitForError(); + void WaitForError(EMessageStatus status); - void SendMessagesWaitReplies(size_t count, const TNetAddr* addr = nullptr); - void SendMessagesWaitReplies(size_t count, const TNetAddr& addr); + void SendMessagesWaitReplies(size_t count, const TNetAddr* addr = nullptr); + void SendMessagesWaitReplies(size_t count, const TNetAddr& addr); - void OnReply(TAutoPtr<TBusMessage> mess, TAutoPtr<TBusMessage> reply) override; + void OnReply(TAutoPtr<TBusMessage> mess, TAutoPtr<TBusMessage> reply) override; - void OnError(TAutoPtr<TBusMessage> mess, EMessageStatus) override; - }; + void OnError(TAutoPtr<TBusMessage> mess, EMessageStatus) override; + }; - class TExampleServer: private TBusServerHandlerError { - public: - TExampleProtocol Proto; - bool UseCompression; - bool AckMessageBeforeSendReply; - TMaybe<size_t> DataSize; // Nothing means use request size - bool ForgetRequest; + class TExampleServer: private TBusServerHandlerError { + public: + TExampleProtocol Proto; + bool UseCompression; + bool AckMessageBeforeSendReply; + TMaybe<size_t> DataSize; // Nothing means use request size + bool ForgetRequest; - TTestSync TestSync; + TTestSync TestSync; - TBusMessageQueuePtr Bus; - TBusServerSessionPtr Session; + TBusMessageQueuePtr Bus; + TBusServerSessionPtr Session; - public: - TExampleServer( - const char* name = "TExampleServer", - const TBusServerSessionConfig& sessionConfig = TBusServerSessionConfig()); + public: + TExampleServer( + const char* name = "TExampleServer", + const TBusServerSessionConfig& sessionConfig = TBusServerSessionConfig()); - TExampleServer(unsigned port, const char* name = "TExampleServer"); + TExampleServer(unsigned port, const char* name = "TExampleServer"); - ~TExampleServer() override; + ~TExampleServer() override; - public: - size_t GetInFlight() const; - unsigned GetActualListenPort() const; - // any of - TNetAddr GetActualListenAddr() const; + public: + size_t GetInFlight() const; + unsigned GetActualListenPort() const; + // any of + TNetAddr GetActualListenAddr() const; - void WaitForOnMessageCount(unsigned n); + void WaitForOnMessageCount(unsigned n); - protected: - void OnMessage(TOnMessageContext& mess) override; - }; + protected: + void OnMessage(TOnMessageContext& mess) override; + }; - } -} + } +} diff --git a/library/cpp/messagebus/test/helper/example_module.cpp b/library/cpp/messagebus/test/helper/example_module.cpp index 65ecfcf73f..26dc184b16 100644 --- a/library/cpp/messagebus/test/helper/example_module.cpp +++ b/library/cpp/messagebus/test/helper/example_module.cpp @@ -31,10 +31,10 @@ TBusServerSessionPtr TExampleServerModule::CreateExtSession(TBusMessageQueue& qu return r; } -TExampleClientModule::TExampleClientModule() - : Source() -{ -} +TExampleClientModule::TExampleClientModule() + : Source() +{ +} TBusServerSessionPtr TExampleClientModule::CreateExtSession(TBusMessageQueue& queue) { Source = CreateDefaultSource(queue, &Proto, TBusServerSessionConfig()); diff --git a/library/cpp/messagebus/test/helper/example_module.h b/library/cpp/messagebus/test/helper/example_module.h index a0b295f613..1f00b25990 100644 --- a/library/cpp/messagebus/test/helper/example_module.h +++ b/library/cpp/messagebus/test/helper/example_module.h @@ -4,34 +4,34 @@ #include <library/cpp/messagebus/oldmodule/module.h> -namespace NBus { - namespace NTest { - struct TExampleModule: public TBusModule { - TExampleProtocol Proto; - TBusMessageQueuePtr Queue; +namespace NBus { + namespace NTest { + struct TExampleModule: public TBusModule { + TExampleProtocol Proto; + TBusMessageQueuePtr Queue; - TExampleModule(); + TExampleModule(); - void StartModule(); + void StartModule(); - bool Shutdown() override; + bool Shutdown() override; - // nop by default - TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override; - }; + // nop by default + TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override; + }; - struct TExampleServerModule: public TExampleModule { - TNetAddr ServerAddr; - TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override; - }; + struct TExampleServerModule: public TExampleModule { + TNetAddr ServerAddr; + TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override; + }; - struct TExampleClientModule: public TExampleModule { - TBusClientSessionPtr Source; + struct TExampleClientModule: public TExampleModule { + TBusClientSessionPtr Source; - TExampleClientModule(); + TExampleClientModule(); - TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override; - }; + TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override; + }; - } -} + } +} diff --git a/library/cpp/messagebus/test/helper/fixed_port.cpp b/library/cpp/messagebus/test/helper/fixed_port.cpp index 258da0d1a5..540fa7c39d 100644 --- a/library/cpp/messagebus/test/helper/fixed_port.cpp +++ b/library/cpp/messagebus/test/helper/fixed_port.cpp @@ -4,7 +4,7 @@ #include <stdlib.h> -bool NBus::NTest::IsFixedPortTestAllowed() { +bool NBus::NTest::IsFixedPortTestAllowed() { // TODO: report skipped tests to test return !GetEnv("MB_TESTS_SKIP_FIXED_PORT"); } diff --git a/library/cpp/messagebus/test/helper/fixed_port.h b/library/cpp/messagebus/test/helper/fixed_port.h index a9c61ebc63..e59d933b2f 100644 --- a/library/cpp/messagebus/test/helper/fixed_port.h +++ b/library/cpp/messagebus/test/helper/fixed_port.h @@ -1,11 +1,11 @@ #pragma once -namespace NBus { - namespace NTest { - bool IsFixedPortTestAllowed(); +namespace NBus { + namespace NTest { + bool IsFixedPortTestAllowed(); - // Must not be in range OS uses for bind on random port. - const unsigned FixedPort = 4927; + // Must not be in range OS uses for bind on random port. + const unsigned FixedPort = 4927; - } -} + } +} diff --git a/library/cpp/messagebus/test/helper/hanging_server.cpp b/library/cpp/messagebus/test/helper/hanging_server.cpp index a35514b00d..b9c2f0571d 100644 --- a/library/cpp/messagebus/test/helper/hanging_server.cpp +++ b/library/cpp/messagebus/test/helper/hanging_server.cpp @@ -4,7 +4,7 @@ using namespace NBus; -THangingServer::THangingServer(int port) { +THangingServer::THangingServer(int port) { BindResult = BindOnPort(port, false); } diff --git a/library/cpp/messagebus/test/helper/hanging_server.h b/library/cpp/messagebus/test/helper/hanging_server.h index cc9fb274d8..2804b81f6f 100644 --- a/library/cpp/messagebus/test/helper/hanging_server.h +++ b/library/cpp/messagebus/test/helper/hanging_server.h @@ -7,7 +7,7 @@ class THangingServer { private: std::pair<unsigned, TVector<NBus::TBindResult>> BindResult; - + public: // listen on given port, and nothing else THangingServer(int port = 0); diff --git a/library/cpp/messagebus/test/helper/message_handler_error.h b/library/cpp/messagebus/test/helper/message_handler_error.h index a314b10761..13097b771d 100644 --- a/library/cpp/messagebus/test/helper/message_handler_error.h +++ b/library/cpp/messagebus/test/helper/message_handler_error.h @@ -2,18 +2,18 @@ #include <library/cpp/messagebus/ybus.h> -namespace NBus { - namespace NTest { - struct TBusClientHandlerError: public IBusClientHandler { - void OnError(TAutoPtr<TBusMessage> pMessage, EMessageStatus status) override; - void OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage) override; - void OnReply(TAutoPtr<TBusMessage> pMessage, TAutoPtr<TBusMessage> pReply) override; - }; +namespace NBus { + namespace NTest { + struct TBusClientHandlerError: public IBusClientHandler { + void OnError(TAutoPtr<TBusMessage> pMessage, EMessageStatus status) override; + void OnMessageSentOneWay(TAutoPtr<TBusMessage> pMessage) override; + void OnReply(TAutoPtr<TBusMessage> pMessage, TAutoPtr<TBusMessage> pReply) override; + }; - struct TBusServerHandlerError: public IBusServerHandler { - void OnError(TAutoPtr<TBusMessage> pMessage, EMessageStatus status) override; - void OnMessage(TOnMessageContext& pMessage) override; - }; + struct TBusServerHandlerError: public IBusServerHandler { + void OnError(TAutoPtr<TBusMessage> pMessage, EMessageStatus status) override; + void OnMessage(TOnMessageContext& pMessage) override; + }; - } -} + } +} diff --git a/library/cpp/messagebus/test/helper/object_count_check.h b/library/cpp/messagebus/test/helper/object_count_check.h index 1c4756e58c..7843fb0f30 100644 --- a/library/cpp/messagebus/test/helper/object_count_check.h +++ b/library/cpp/messagebus/test/helper/object_count_check.h @@ -1,7 +1,7 @@ #pragma once #include <library/cpp/testing/unittest/registar.h> - + #include <library/cpp/messagebus/remote_client_connection.h> #include <library/cpp/messagebus/remote_client_session.h> #include <library/cpp/messagebus/remote_server_connection.h> @@ -23,15 +23,15 @@ struct TObjectCountCheck { struct TReset { TObjectCountCheck* const Thiz; - TReset(TObjectCountCheck* thiz) - : Thiz(thiz) - { - } + TReset(TObjectCountCheck* thiz) + : Thiz(thiz) + { + } - void operator()() { + void operator()() { long oldValue = TObjectCounter<T>::ResetObjectCount(); if (oldValue != 0) { - Cerr << "warning: previous counter: " << oldValue << " for " << TypeName<T>() << Endl; + Cerr << "warning: previous counter: " << oldValue << " for " << TypeName<T>() << Endl; Cerr << "won't check in this test" << Endl; Thiz->Enabled = false; } @@ -45,10 +45,10 @@ struct TObjectCountCheck { template <typename T> struct TCheckZero { - TCheckZero(TObjectCountCheck*) { - } + TCheckZero(TObjectCountCheck*) { + } - void operator()() { + void operator()() { UNIT_ASSERT_VALUES_EQUAL_C(0L, TObjectCounter<T>::ObjectCount(), TypeName<T>()); } }; diff --git a/library/cpp/messagebus/test/helper/wait_for.h b/library/cpp/messagebus/test/helper/wait_for.h index f09958d4c0..029ab0da48 100644 --- a/library/cpp/messagebus/test/helper/wait_for.h +++ b/library/cpp/messagebus/test/helper/wait_for.h @@ -3,12 +3,12 @@ #include <util/datetime/base.h> #include <util/system/yassert.h> -#define UNIT_WAIT_FOR(condition) \ - do { \ +#define UNIT_WAIT_FOR(condition) \ + do { \ TInstant start(TInstant::Now()); \ while (!(condition) && (TInstant::Now() - start < TDuration::Seconds(10))) { \ - Sleep(TDuration::MilliSeconds(1)); \ + Sleep(TDuration::MilliSeconds(1)); \ } \ /* TODO: use UNIT_ASSERT if in unittest thread */ \ - Y_VERIFY(condition, "condition failed after 10 seconds wait"); \ + Y_VERIFY(condition, "condition failed after 10 seconds wait"); \ } while (0) diff --git a/library/cpp/messagebus/test/perftest/perftest.cpp b/library/cpp/messagebus/test/perftest/perftest.cpp index 8489319278..8ce4c175a2 100644 --- a/library/cpp/messagebus/test/perftest/perftest.cpp +++ b/library/cpp/messagebus/test/perftest/perftest.cpp @@ -41,18 +41,18 @@ using namespace NBus; const int DEFAULT_PORT = 55666; struct TPerftestConfig { - TString Nodes; ///< node1:port1,node2:port2 - int ClientCount; - int MessageSize; ///< size of message to send - int Delay; ///< server delay (milliseconds) - float Failure; ///< simulated failure rate - int ServerPort; - int Run; - bool ServerUseModules; - bool ExecuteOnMessageInWorkerPool; - bool ExecuteOnReplyInWorkerPool; - bool UseCompression; - bool Profile; + TString Nodes; ///< node1:port1,node2:port2 + int ClientCount; + int MessageSize; ///< size of message to send + int Delay; ///< server delay (milliseconds) + float Failure; ///< simulated failure rate + int ServerPort; + int Run; + bool ServerUseModules; + bool ExecuteOnMessageInWorkerPool; + bool ExecuteOnReplyInWorkerPool; + bool UseCompression; + bool Profile; unsigned WwwPort; TPerftestConfig(); @@ -61,8 +61,8 @@ struct TPerftestConfig { fprintf(stderr, "ClientCount=%d\n", ClientCount); fprintf(stderr, "ServerPort=%d\n", ServerPort); fprintf(stderr, "Delay=%d usecs\n", Delay); - fprintf(stderr, "MessageSize=%d bytes\n", MessageSize); - fprintf(stderr, "Failure=%.3f%%\n", Failure * 100.0); + fprintf(stderr, "MessageSize=%d bytes\n", MessageSize); + fprintf(stderr, "Failure=%.3f%%\n", Failure * 100.0); fprintf(stderr, "Runtime=%d seconds\n", Run); fprintf(stderr, "ServerUseModules=%s\n", ServerUseModules ? "true" : "false"); fprintf(stderr, "ExecuteOnMessageInWorkerPool=%s\n", ExecuteOnMessageInWorkerPool ? "true" : "false"); @@ -73,7 +73,7 @@ struct TPerftestConfig { } }; -extern TPerftestConfig* TheConfig; +extern TPerftestConfig* TheConfig; extern bool TheExit; TVector<TNetAddr> ServerAddresses; @@ -191,26 +191,26 @@ struct TTestStats { TAtomic Errors; TAtomic Replies; - void IncMessage() { - AtomicIncrement(Messages); - } - void IncReplies() { - AtomicDecrement(Messages); - AtomicIncrement(Replies); - } - int NumMessage() { - return AtomicGet(Messages); - } - void IncErrors() { - AtomicDecrement(Messages); - AtomicIncrement(Errors); - } - int NumErrors() { - return AtomicGet(Errors); - } - int NumReplies() { - return AtomicGet(Replies); - } + void IncMessage() { + AtomicIncrement(Messages); + } + void IncReplies() { + AtomicDecrement(Messages); + AtomicIncrement(Replies); + } + int NumMessage() { + return AtomicGet(Messages); + } + void IncErrors() { + AtomicDecrement(Messages); + AtomicIncrement(Errors); + } + int NumErrors() { + return AtomicGet(Errors); + } + int NumReplies() { + return AtomicGet(Replies); + } double GetThroughput() { return NumReplies() * 1000000.0 / (TInstant::Now() - Start).MicroSeconds(); @@ -232,7 +232,7 @@ TTestStats Stats; //////////////////////////////////////////////////////////////////// /// \brief Fast of the client session -class TPerftestClient : IBusClientHandler { +class TPerftestClient : IBusClientHandler { public: TBusClientSessionPtr Session; THolder<TBusProtocol> Proto; @@ -270,7 +270,7 @@ public: connection = Connections.at(RandomNumber<size_t>()).Get(); } - TBusMessage* message = NewRequest().Release(); + TBusMessage* message = NewRequest().Release(); int ret = connection->SendMessage(message, true); if (ret == MESSAGE_OK) { @@ -386,7 +386,7 @@ public: CheckRequest(typed); /// forget replies for few messages, see what happends - if (TheConfig->Failure > RandomNumber<double>()) { + if (TheConfig->Failure > RandomNumber<double>()) { return; } @@ -420,7 +420,7 @@ public: Y_VERIFY(StartInput(), "failed to start input"); } - ~TPerftestUsingModule() override { + ~TPerftestUsingModule() override { Shutdown(); } @@ -435,7 +435,7 @@ private: } /// forget replies for few messages, see what happends - if (TheConfig->Failure > RandomNumber<double>()) { + if (TheConfig->Failure > RandomNumber<double>()) { return nullptr; } @@ -454,15 +454,15 @@ using namespace std; using namespace NBus; static TNetworkAddress ParseNetworkAddress(const char* string) { - TString Name; - int Port; + TString Name; + int Port; - const char* port = strchr(string, ':'); + const char* port = strchr(string, ':'); if (port != nullptr) { Name.append(string, port - string); Port = atoi(port + 1); - } else { + } else { Name.append(string); Port = TheConfig->ServerPort != 0 ? TheConfig->ServerPort : DEFAULT_PORT; } @@ -503,19 +503,19 @@ TPerftestConfig::TPerftestConfig() { WwwPort = 0; } -TPerftestConfig* TheConfig = new TPerftestConfig(); -bool TheExit = false; +TPerftestConfig* TheConfig = new TPerftestConfig(); +bool TheExit = false; TSystemEvent StopEvent; -TSimpleSharedPtr<TPerftestServer> Server; -TSimpleSharedPtr<TPerftestUsingModule> ServerUsingModule; +TSimpleSharedPtr<TPerftestServer> Server; +TSimpleSharedPtr<TPerftestUsingModule> ServerUsingModule; -TVector<TSimpleSharedPtr<TPerftestClient>> Clients; +TVector<TSimpleSharedPtr<TPerftestClient>> Clients; TMutex ClientsLock; void stopsignal(int /*sig*/) { - fprintf(stderr, "\n-------------------- exiting ------------------\n"); + fprintf(stderr, "\n-------------------- exiting ------------------\n"); TheExit = true; StopEvent.Signal(); } @@ -531,22 +531,22 @@ void TTestStats::PeriodicallyPrint() { if (TheExit) break; - TVector<TSimpleSharedPtr<TPerftestClient>> clients; + TVector<TSimpleSharedPtr<TPerftestClient>> clients; { TGuard<TMutex> guard(ClientsLock); clients = Clients; } fprintf(stderr, "replies=%d errors=%d throughput=%.3f mess/sec\n", - NumReplies(), NumErrors(), GetThroughput()); + NumReplies(), NumErrors(), GetThroughput()); if (!!Server) { fprintf(stderr, "server: q: %u %s\n", - (unsigned)Server->Bus->GetExecutor()->GetWorkQueueSize(), + (unsigned)Server->Bus->GetExecutor()->GetWorkQueueSize(), Server->Session->GetStatusSingleLine().data()); } if (!!ServerUsingModule) { fprintf(stderr, "server: q: %u %s\n", - (unsigned)ServerUsingModule->Bus->GetExecutor()->GetWorkQueueSize(), + (unsigned)ServerUsingModule->Bus->GetExecutor()->GetWorkQueueSize(), ServerUsingModule->Session->GetStatusSingleLine().data()); } for (const auto& client : clients) { @@ -587,19 +587,19 @@ void TTestStats::PeriodicallyPrint() { } } -int main(int argc, char* argv[]) { +int main(int argc, char* argv[]) { NLWTrace::StartLwtraceFromEnv(); - /* unix foo */ + /* unix foo */ setvbuf(stdout, nullptr, _IONBF, 0); setvbuf(stderr, nullptr, _IONBF, 0); Umask(0); - SetAsyncSignalHandler(SIGINT, stopsignal); + SetAsyncSignalHandler(SIGINT, stopsignal); SetAsyncSignalHandler(SIGTERM, stopsignal); #ifndef _win_ SetAsyncSignalHandler(SIGUSR1, stopsignal); #endif - signal(SIGPIPE, SIG_IGN); + signal(SIGPIPE, SIG_IGN); NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default(); opts.AddLongOption('s', "server-port", "server port").RequiredArgument("port").StoreResult(&TheConfig->ServerPort); @@ -611,11 +611,11 @@ int main(int argc, char* argv[]) { opts.AddLongOption("client-count", "amount of clients").RequiredArgument("count").StoreResult(&TheConfig->ClientCount).DefaultValue("1"); opts.AddLongOption("server-use-modules").StoreResult(&TheConfig->ServerUseModules, true); opts.AddLongOption("on-message-in-pool", "execute OnMessage callback in worker pool") - .RequiredArgument("BOOL") - .StoreResult(&TheConfig->ExecuteOnMessageInWorkerPool); + .RequiredArgument("BOOL") + .StoreResult(&TheConfig->ExecuteOnMessageInWorkerPool); opts.AddLongOption("on-reply-in-pool", "execute OnReply callback in worker pool") - .RequiredArgument("BOOL") - .StoreResult(&TheConfig->ExecuteOnReplyInWorkerPool); + .RequiredArgument("BOOL") + .StoreResult(&TheConfig->ExecuteOnReplyInWorkerPool); opts.AddLongOption("compression", "use compression").RequiredArgument("BOOL").StoreResult(&TheConfig->UseCompression); opts.AddLongOption("simple-proto").SetFlag(&Config.SimpleProtocol); opts.AddLongOption("profile").SetFlag(&TheConfig->Profile); @@ -651,8 +651,8 @@ int main(int argc, char* argv[]) { www->RegisterServerSession(Server->Session); } } - - TVector<TSimpleSharedPtr<NThreading::TLegacyFuture<void, false>>> futures; + + TVector<TSimpleSharedPtr<NThreading::TLegacyFuture<void, false>>> futures; if (ServerAddresses.size() > 0 && TheConfig->ClientCount > 0) { for (int i = 0; i < TheConfig->ClientCount; ++i) { @@ -684,7 +684,7 @@ int main(int argc, char* argv[]) { ServerUsingModule->Stop(); } - TVector<TSimpleSharedPtr<TPerftestClient>> clients; + TVector<TSimpleSharedPtr<TPerftestClient>> clients; { TGuard<TMutex> guard(ClientsLock); clients = Clients; diff --git a/library/cpp/messagebus/test/perftest/simple_proto.cpp b/library/cpp/messagebus/test/perftest/simple_proto.cpp index 19d6c15b9d..a54d4b3493 100644 --- a/library/cpp/messagebus/test/perftest/simple_proto.cpp +++ b/library/cpp/messagebus/test/perftest/simple_proto.cpp @@ -6,10 +6,10 @@ using namespace NBus; -void TSimpleProtocol::Serialize(const TBusMessage* mess, TBuffer& data) { +void TSimpleProtocol::Serialize(const TBusMessage* mess, TBuffer& data) { Y_VERIFY(typeid(TSimpleMessage) == typeid(*mess)); const TSimpleMessage* typed = static_cast<const TSimpleMessage*>(mess); - data.Append((const char*)&typed->Payload, 4); + data.Append((const char*)&typed->Payload, 4); } TAutoPtr<TBusMessage> TSimpleProtocol::Deserialize(ui16, TArrayRef<const char> payload) { diff --git a/library/cpp/messagebus/test/perftest/simple_proto.h b/library/cpp/messagebus/test/perftest/simple_proto.h index 4a0cc08db3..b61c4f4ae6 100644 --- a/library/cpp/messagebus/test/perftest/simple_proto.h +++ b/library/cpp/messagebus/test/perftest/simple_proto.h @@ -2,28 +2,28 @@ #include <library/cpp/messagebus/ybus.h> -struct TSimpleMessage: public NBus::TBusMessage { +struct TSimpleMessage: public NBus::TBusMessage { ui32 Payload; TSimpleMessage() - : TBusMessage(1) - , Payload(0) - { - } + : TBusMessage(1) + , Payload(0) + { + } TSimpleMessage(NBus::ECreateUninitialized) : TBusMessage(NBus::ECreateUninitialized()) - { - } + { + } }; struct TSimpleProtocol: public NBus::TBusProtocol { - TSimpleProtocol() - : NBus::TBusProtocol("simple", 55666) - { - } + TSimpleProtocol() + : NBus::TBusProtocol("simple", 55666) + { + } - void Serialize(const NBus::TBusMessage* mess, TBuffer& data) override; + void Serialize(const NBus::TBusMessage* mess, TBuffer& data) override; TAutoPtr<NBus::TBusMessage> Deserialize(ui16 ty, TArrayRef<const char> payload) override; }; diff --git a/library/cpp/messagebus/test/ut/count_down_latch.h b/library/cpp/messagebus/test/ut/count_down_latch.h index 5117db5731..a4d6b72bfa 100644 --- a/library/cpp/messagebus/test/ut/count_down_latch.h +++ b/library/cpp/messagebus/test/ut/count_down_latch.h @@ -7,12 +7,12 @@ class TCountDownLatch { private: TAtomic Current; TSystemEvent EventObject; - + public: TCountDownLatch(unsigned initial) : Current(initial) - { - } + { + } void CountDown() { if (AtomicDecrement(Current) == 0) { diff --git a/library/cpp/messagebus/test/ut/locator_uniq_ut.cpp b/library/cpp/messagebus/test/ut/locator_uniq_ut.cpp index 3fdd175d73..dd5dfc4cca 100644 --- a/library/cpp/messagebus/test/ut/locator_uniq_ut.cpp +++ b/library/cpp/messagebus/test/ut/locator_uniq_ut.cpp @@ -4,12 +4,12 @@ #include <library/cpp/messagebus/ybus.h> class TLocatorRegisterUniqTest: public TTestBase { - UNIT_TEST_SUITE(TLocatorRegisterUniqTest); - UNIT_TEST(TestRegister); - UNIT_TEST_SUITE_END(); + UNIT_TEST_SUITE(TLocatorRegisterUniqTest); + UNIT_TEST(TestRegister); + UNIT_TEST_SUITE_END(); -protected: - void TestRegister(); +protected: + void TestRegister(); }; UNIT_TEST_SUITE_REGISTRATION(TLocatorRegisterUniqTest); diff --git a/library/cpp/messagebus/test/ut/messagebus_ut.cpp b/library/cpp/messagebus/test/ut/messagebus_ut.cpp index 040f9b7702..92839e9cf9 100644 --- a/library/cpp/messagebus/test/ut/messagebus_ut.cpp +++ b/library/cpp/messagebus/test/ut/messagebus_ut.cpp @@ -10,8 +10,8 @@ #include <util/network/sock.h> -#include <utility> - +#include <utility> + using namespace NBus; using namespace NBus::NTest; @@ -23,10 +23,10 @@ namespace { TExampleClientSlowOnMessageSent() : SentCompleted(0) - { - } + { + } - ~TExampleClientSlowOnMessageSent() override { + ~TExampleClientSlowOnMessageSent() override { Session->Shutdown(); } @@ -48,7 +48,7 @@ namespace { Y_UNIT_TEST_SUITE(TMessageBusTests) { void TestDestinationTemplate(bool useCompression, bool ackMessageBeforeReply, - const TBusServerSessionConfig& sessionConfig) { + const TBusServerSessionConfig& sessionConfig) { TObjectCountCheck objectCountCheck; TExampleServer server; @@ -121,17 +121,17 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { client.SendMessagesWaitReplies(19, serverAddr); } - struct TestNoServerImplClient: public TExampleClient { + struct TestNoServerImplClient: public TExampleClient { TTestSync TestSync; int failures = 0; - template <typename... Args> - TestNoServerImplClient(Args&&... args) - : TExampleClient(std::forward<Args>(args)...) - { - } + template <typename... Args> + TestNoServerImplClient(Args&&... args) + : TExampleClient(std::forward<Args>(args)...) + { + } - ~TestNoServerImplClient() override { + ~TestNoServerImplClient() override { Session->Shutdown(); } @@ -155,7 +155,7 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { if (oneWay) { status = client.Session->SendMessageOneWay(new TExampleRequest(&client.Proto.RequestCount), &noServerAddr); } else { - TAutoPtr<TBusMessage> message(new TExampleRequest(&client.Proto.RequestCount)); + TAutoPtr<TBusMessage> message(new TExampleRequest(&client.Proto.RequestCount)); status = client.Session->SendMessageAutoPtr(message, &noServerAddr); } @@ -168,7 +168,7 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { client.TestSync.WaitForAndIncrement(count * 2 + 1); } - client.TestSync.WaitForAndIncrement(count * 2); + client.TestSync.WaitForAndIncrement(count * 2); } void HangingServerImpl(unsigned port) { @@ -241,7 +241,7 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { client.WaitReplies(); } - struct TSendTimeoutCheckerExampleClient: public TExampleClient { + struct TSendTimeoutCheckerExampleClient: public TExampleClient { static TBusClientSessionConfig SessionConfig(bool periodLessThanConnectTimeout) { TBusClientSessionConfig sessionConfig; if (periodLessThanConnectTimeout) { @@ -256,8 +256,8 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { TSendTimeoutCheckerExampleClient(bool periodLessThanConnectTimeout) : TExampleClient(SessionConfig(periodLessThanConnectTimeout)) - { - } + { + } ~TSendTimeoutCheckerExampleClient() override { Session->Shutdown(); @@ -470,7 +470,7 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { client.WaitForError(MESSAGE_MESSAGE_TOO_LARGE); } - struct TServerForResponseTooLarge: public TExampleServer { + struct TServerForResponseTooLarge: public TExampleServer { TTestSync TestSync; static TBusServerSessionConfig Config() { @@ -481,10 +481,10 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { TServerForResponseTooLarge() : TExampleServer("TServerForResponseTooLarge", Config()) - { - } + { + } - ~TServerForResponseTooLarge() override { + ~TServerForResponseTooLarge() override { Session->Shutdown(); } @@ -530,7 +530,7 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { UNIT_ASSERT_VALUES_EQUAL(1, client.Session->GetInFlight()); } - struct TServerForRequestTooLarge: public TExampleServer { + struct TServerForRequestTooLarge: public TExampleServer { TTestSync TestSync; static TBusServerSessionConfig Config() { @@ -541,10 +541,10 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { TServerForRequestTooLarge() : TExampleServer("TServerForRequestTooLarge", Config()) - { - } + { + } - ~TServerForRequestTooLarge() override { + ~TServerForRequestTooLarge() override { Session->Shutdown(); } @@ -674,7 +674,7 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { client.WaitReplies(); } - struct TResetAfterSendOneWayErrorInCallbackClient: public TExampleClient { + struct TResetAfterSendOneWayErrorInCallbackClient: public TExampleClient { TTestSync TestSync; static TBusClientSessionConfig SessionConfig() { @@ -691,7 +691,7 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { { } - ~TResetAfterSendOneWayErrorInCallbackClient() override { + ~TResetAfterSendOneWayErrorInCallbackClient() override { Session->Shutdown(); } @@ -716,10 +716,10 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { client.TestSync.WaitForAndIncrement(2); } - struct TResetAfterSendMessageOneWayDuringShutdown: public TExampleClient { + struct TResetAfterSendMessageOneWayDuringShutdown: public TExampleClient { TTestSync TestSync; - ~TResetAfterSendMessageOneWayDuringShutdown() override { + ~TResetAfterSendMessageOneWayDuringShutdown() override { Session->Shutdown(); } @@ -770,10 +770,10 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { TestNoServerImpl(17, true); } - struct TResetAfterSendOneWaySuccessClient: public TExampleClient { + struct TResetAfterSendOneWaySuccessClient: public TExampleClient { TTestSync TestSync; - ~TResetAfterSendOneWaySuccessClient() override { + ~TResetAfterSendOneWaySuccessClient() override { Session->Shutdown(); } @@ -835,7 +835,7 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { TExampleProtocol proto; TBusServerHandlerError handler; TBusServerSessionPtr session = TBusServerSession::Create( - &proto, &handler, TBusServerSessionConfig(), queue); + &proto, &handler, TBusServerSessionConfig(), queue); unsigned port = session->GetActualListenPort(); UNIT_ASSERT(port > 0); @@ -873,7 +873,7 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { size_t pos = 0; while (pos < sizeof(response)) { - size_t count = input.Read(((char*)&response) + pos, sizeof(response) - pos); + size_t count = input.Read(((char*)&response) + pos, sizeof(response) - pos); pos += count; } @@ -882,10 +882,10 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { UNIT_ASSERT_VALUES_EQUAL(YBUS_VERSION, response.GetVersionInternal()); } - struct TOnConnectionEventClient: public TExampleClient { + struct TOnConnectionEventClient: public TExampleClient { TTestSync Sync; - ~TOnConnectionEventClient() override { + ~TOnConnectionEventClient() override { Session->Shutdown(); } @@ -913,13 +913,13 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { } }; - struct TOnConnectionEventServer: public TExampleServer { + struct TOnConnectionEventServer: public TExampleServer { TOnConnectionEventServer() - : TExampleServer("TOnConnectionEventServer") - { - } + : TExampleServer("TOnConnectionEventServer") + { + } - ~TOnConnectionEventServer() override { + ~TOnConnectionEventServer() override { Session->Shutdown(); } @@ -963,9 +963,9 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { client.Sync.WaitForAndIncrement(3); } - struct TServerForQuotaWake: public TExampleServer { + struct TServerForQuotaWake: public TExampleServer { TSystemEvent GoOn; - TMutex OneLock; + TMutex OneLock; TOnMessageContext OneMessage; @@ -981,16 +981,16 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { TServerForQuotaWake() : TExampleServer("TServerForQuotaWake", Config()) - { - } + { + } - ~TServerForQuotaWake() override { + ~TServerForQuotaWake() override { Session->Shutdown(); } void OnMessage(TOnMessageContext& req) override { if (!GoOn.Wait(0)) { - TGuard<TMutex> guard(OneLock); + TGuard<TMutex> guard(OneLock); UNIT_ASSERT(!OneMessage); @@ -1000,7 +1000,7 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { } void WakeOne() { - TGuard<TMutex> guard(OneLock); + TGuard<TMutex> guard(OneLock); UNIT_ASSERT(!!OneMessage); @@ -1035,13 +1035,13 @@ Y_UNIT_TEST_SUITE(TMessageBusTests) { count++; } else if (status == MESSAGE_BUSY) { - if (count == test_msg_count) { + if (count == test_msg_count) { TInstant now = TInstant::Now(); - if (start.GetValue() == 0) { + if (start.GetValue() == 0) { start = now; - // TODO: properly check that server is blocked + // TODO: properly check that server is blocked } else if (start + TDuration::MilliSeconds(100) < now) { break; } diff --git a/library/cpp/messagebus/test/ut/module_client_one_way_ut.cpp b/library/cpp/messagebus/test/ut/module_client_one_way_ut.cpp index 4083cf3b7b..fd511e2dd9 100644 --- a/library/cpp/messagebus/test/ut/module_client_one_way_ut.cpp +++ b/library/cpp/messagebus/test/ut/module_client_one_way_ut.cpp @@ -43,8 +43,8 @@ Y_UNIT_TEST_SUITE(ModuleClientOneWay) { : TBusModule("m") , TestSync(testSync) , Port(port) - { - } + { + } TJobHandler Start(TBusJob* job, TBusMessage*) override { TestSync->WaitForAndIncrement(0); @@ -94,8 +94,8 @@ Y_UNIT_TEST_SUITE(ModuleClientOneWay) { TSendErrorModule(TTestSync* testSync) : TBusModule("m") , TestSync(testSync) - { - } + { + } TJobHandler Start(TBusJob* job, TBusMessage*) override { TestSync->WaitForAndIncrement(0); diff --git a/library/cpp/messagebus/test/ut/module_client_ut.cpp b/library/cpp/messagebus/test/ut/module_client_ut.cpp index ebfe185cc6..84897ce5c4 100644 --- a/library/cpp/messagebus/test/ut/module_client_ut.cpp +++ b/library/cpp/messagebus/test/ut/module_client_ut.cpp @@ -20,34 +20,34 @@ using namespace NBus::NTest; // helper class that cleans TBusJob instance, so job's destructor can // be completed without assertion fail. struct TJobGuard { -public: - TJobGuard(NBus::TBusJob* job) - : Job(job) - { - } - - ~TJobGuard() { - Job->ClearAllMessageStates(); - } - -private: - NBus::TBusJob* Job; +public: + TJobGuard(NBus::TBusJob* job) + : Job(job) + { + } + + ~TJobGuard() { + Job->ClearAllMessageStates(); + } + +private: + NBus::TBusJob* Job; }; -class TMessageOk: public NBus::TBusMessage { -public: - TMessageOk() - : NBus::TBusMessage(1) - { - } +class TMessageOk: public NBus::TBusMessage { +public: + TMessageOk() + : NBus::TBusMessage(1) + { + } }; -class TMessageError: public NBus::TBusMessage { -public: - TMessageError() - : NBus::TBusMessage(2) - { - } +class TMessageError: public NBus::TBusMessage { +public: + TMessageError() + : NBus::TBusMessage(2) + { + } }; Y_UNIT_TEST_SUITE(BusJobTest) { @@ -108,8 +108,8 @@ Y_UNIT_TEST_SUITE(BusJobTest) { TParallelOnReplyModule(const TNetAddr& serverAddr) : ServerAddr(serverAddr) , RepliesLatch(2) - { - } + { + } TJobHandler Start(TBusJob* job, TBusMessage* mess) override { Y_UNUSED(mess); @@ -166,8 +166,8 @@ Y_UNIT_TEST_SUITE(BusJobTest) { : ServerAddr("localhost", 17) , GotReplyLatch(2) , SentMessage() - { - } + { + } TJobHandler Start(TBusJob* job, TBusMessage* mess) override { Y_UNUSED(mess); @@ -222,7 +222,7 @@ Y_UNIT_TEST_SUITE(BusJobTest) { module.Shutdown(); } - struct TSlowReplyServer: public TBusServerHandlerError { + struct TSlowReplyServer: public TBusServerHandlerError { TTestSync* const TestSync; TBusMessageQueuePtr Bus; TBusServerSessionPtr ServerSession; @@ -248,7 +248,7 @@ Y_UNIT_TEST_SUITE(BusJobTest) { } }; - struct TModuleThatSendsReplyEarly: public TExampleClientModule { + struct TModuleThatSendsReplyEarly: public TExampleClientModule { TTestSync* const TestSync; const unsigned ServerPort; @@ -260,8 +260,8 @@ Y_UNIT_TEST_SUITE(BusJobTest) { , ServerPort(serverPort) , ServerSession(nullptr) , ReplyCount(0) - { - } + { + } TJobHandler Start(TBusJob* job, TBusMessage* mess) override { Y_UNUSED(mess); @@ -318,22 +318,22 @@ Y_UNIT_TEST_SUITE(BusJobTest) { module.Shutdown(); } - struct TShutdownCalledBeforeReplyReceivedModule: public TExampleClientModule { + struct TShutdownCalledBeforeReplyReceivedModule: public TExampleClientModule { unsigned ServerPort; TTestSync TestSync; TShutdownCalledBeforeReplyReceivedModule(unsigned serverPort) : ServerPort(serverPort) - { - } + { + } TJobHandler Start(TBusJob* job, TBusMessage*) override { TestSync.CheckAndIncrement(0); job->Send(new TExampleRequest(&Proto.RequestCount), Source, - TReplyHandler(&TShutdownCalledBeforeReplyReceivedModule::HandleReply), - 0, TNetAddr("localhost", ServerPort)); + TReplyHandler(&TShutdownCalledBeforeReplyReceivedModule::HandleReply), + 0, TNetAddr("localhost", ServerPort)); return &TShutdownCalledBeforeReplyReceivedModule::End; } diff --git a/library/cpp/messagebus/test/ut/module_server_ut.cpp b/library/cpp/messagebus/test/ut/module_server_ut.cpp index 88fe1dd9b6..4258ae4bf7 100644 --- a/library/cpp/messagebus/test/ut/module_server_ut.cpp +++ b/library/cpp/messagebus/test/ut/module_server_ut.cpp @@ -21,7 +21,7 @@ Y_UNIT_TEST_SUITE(ModuleServerTests) { /// create or get instance of message queue, need one per application TBusMessageQueuePtr bus(CreateMessageQueue()); - THostInfoHandler hostHandler(bus.Get()); + THostInfoHandler hostHandler(bus.Get()); TDupDetectModule module(hostHandler.GetActualListenAddr()); bool success; success = module.Init(bus.Get()); @@ -39,13 +39,13 @@ Y_UNIT_TEST_SUITE(ModuleServerTests) { dupHandler.DupDetect->Shutdown(); } - struct TParallelOnMessageModule: public TExampleServerModule { + struct TParallelOnMessageModule: public TExampleServerModule { TCountDownLatch WaitTwoRequestsLatch; TParallelOnMessageModule() : WaitTwoRequestsLatch(2) - { - } + { + } TJobHandler Start(TBusJob* job, TBusMessage* mess) override { WaitTwoRequestsLatch.CountDown(); diff --git a/library/cpp/messagebus/test/ut/moduletest.h b/library/cpp/messagebus/test/ut/moduletest.h index d5da72c0cb..e67da02701 100644 --- a/library/cpp/messagebus/test/ut/moduletest.h +++ b/library/cpp/messagebus/test/ut/moduletest.h @@ -11,211 +11,211 @@ #include <library/cpp/messagebus/ybus.h> #include <library/cpp/messagebus/oldmodule/module.h> -namespace NBus { - namespace NTest { - using namespace std; +namespace NBus { + namespace NTest { + using namespace std; -#define TYPE_HOSTINFOREQUEST 100 +#define TYPE_HOSTINFOREQUEST 100 #define TYPE_HOSTINFORESPONSE 101 - //////////////////////////////////////////////////////////////////// - /// \brief DupDetect protocol that common between client and server - //////////////////////////////////////////////////////////////////// - /// \brief HostInfo request class - class THostInfoMessage: public TBusMessage { - public: - THostInfoMessage() - : TBusMessage(TYPE_HOSTINFOREQUEST) - { - } - THostInfoMessage(ECreateUninitialized) - : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) - { - } - - ~THostInfoMessage() override { - } - }; - - //////////////////////////////////////////////////////////////////// - /// \brief HostInfo reply class - class THostInfoReply: public TBusMessage { - public: - THostInfoReply() - : TBusMessage(TYPE_HOSTINFORESPONSE) - { - } - THostInfoReply(ECreateUninitialized) - : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) - { - } - - ~THostInfoReply() override { - } - }; - - //////////////////////////////////////////////////////////////////// - /// \brief HostInfo protocol that common between client and server - class THostInfoProtocol: public TBusProtocol { - public: - THostInfoProtocol() - : TBusProtocol("HOSTINFO", 0) - { - } - /// serialized protocol specific data into TBusData - void Serialize(const TBusMessage* mess, TBuffer& data) override { - Y_UNUSED(data); - Y_UNUSED(mess); - } - - /// deserialized TBusData into new instance of the message - TAutoPtr<TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) override { - Y_UNUSED(payload); - - if (messageType == TYPE_HOSTINFOREQUEST) { - return new THostInfoMessage(MESSAGE_CREATE_UNINITIALIZED); - } else if (messageType == TYPE_HOSTINFORESPONSE) { - return new THostInfoReply(MESSAGE_CREATE_UNINITIALIZED); - } else { - Y_FAIL("unknown"); - } - } - }; - - ////////////////////////////////////////////////////////////// - /// \brief HostInfo handler (should convert it to module too) - struct THostInfoHandler: public TBusServerHandlerError { - TBusServerSessionPtr Session; - TBusServerSessionConfig HostInfoConfig; - THostInfoProtocol HostInfoProto; - - THostInfoHandler(TBusMessageQueue* queue) { - Session = TBusServerSession::Create(&HostInfoProto, this, HostInfoConfig, queue); - } - - void OnMessage(TOnMessageContext& mess) override { - usleep(10 * 1000); /// pretend we are doing something - - TAutoPtr<THostInfoReply> reply(new THostInfoReply()); - - mess.SendReplyMove(reply); - } - - TNetAddr GetActualListenAddr() { - return TNetAddr("localhost", Session->GetActualListenPort()); - } - }; - - ////////////////////////////////////////////////////////////// - /// \brief DupDetect handler (should convert it to module too) - struct TDupDetectHandler: public TBusClientHandlerError { - TNetAddr ServerAddr; - - TBusClientSessionPtr DupDetect; - TBusClientSessionConfig DupDetectConfig; - TExampleProtocol DupDetectProto; - - int NumMessages; - int NumReplies; - - TDupDetectHandler(const TNetAddr& serverAddr, TBusMessageQueuePtr queue) - : ServerAddr(serverAddr) - { - DupDetect = TBusClientSession::Create(&DupDetectProto, this, DupDetectConfig, queue); - DupDetect->RegisterService("localhost"); - } - - void Work() { - NumMessages = 10; - NumReplies = 0; - - for (int i = 0; i < NumMessages; i++) { - TExampleRequest* mess = new TExampleRequest(&DupDetectProto.RequestCount); - DupDetect->SendMessage(mess, &ServerAddr); - } - } - - void OnReply(TAutoPtr<TBusMessage> mess, TAutoPtr<TBusMessage> reply) override { - Y_UNUSED(mess); - Y_UNUSED(reply); - NumReplies++; - } - }; - - ///////////////////////////////////////////////////////////////// - /// \brief DupDetect module - - struct TDupDetectModule: public TBusModule { - TNetAddr HostInfoAddr; - - TBusClientSessionPtr HostInfoClientSession; - TBusClientSessionConfig HostInfoConfig; - THostInfoProtocol HostInfoProto; - - TExampleProtocol DupDetectProto; - TBusServerSessionConfig DupDetectConfig; - - TNetAddr ListenAddr; - - TDupDetectModule(const TNetAddr& hostInfoAddr) - : TBusModule("DUPDETECTMODULE") - , HostInfoAddr(hostInfoAddr) - { - } - - bool Init(TBusMessageQueue* queue) { - HostInfoClientSession = CreateDefaultSource(*queue, &HostInfoProto, HostInfoConfig); - HostInfoClientSession->RegisterService("localhost"); - - return TBusModule::CreatePrivateSessions(queue); - } - - TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override { - TBusServerSessionPtr session = CreateDefaultDestination(queue, &DupDetectProto, DupDetectConfig); - - ListenAddr = TNetAddr("localhost", session->GetActualListenPort()); - - return session; - } - - /// entry point into module, first function to call - TJobHandler Start(TBusJob* job, TBusMessage* mess) override { - TExampleRequest* dmess = dynamic_cast<TExampleRequest*>(mess); - Y_UNUSED(dmess); - - THostInfoMessage* hmess = new THostInfoMessage(); - - /// send message to imaginary hostinfo server - job->Send(hmess, HostInfoClientSession, TReplyHandler(), 0, HostInfoAddr); - - return TJobHandler(&TDupDetectModule::ProcessHostInfo); - } - - /// next handler is executed when all outstanding requests from previous handler is completed - TJobHandler ProcessHostInfo(TBusJob* job, TBusMessage* mess) { - TExampleRequest* dmess = dynamic_cast<TExampleRequest*>(mess); - Y_UNUSED(dmess); - - THostInfoMessage* hmess = job->Get<THostInfoMessage>(); - THostInfoReply* hreply = job->Get<THostInfoReply>(); - EMessageStatus hstatus = job->GetStatus<THostInfoMessage>(); - Y_ASSERT(hmess != nullptr); - Y_ASSERT(hreply != nullptr); - Y_ASSERT(hstatus == MESSAGE_OK); - - return TJobHandler(&TDupDetectModule::Finish); - } - - /// last handler sends reply and returns NULL - TJobHandler Finish(TBusJob* job, TBusMessage* mess) { - Y_UNUSED(mess); - - TExampleResponse* reply = new TExampleResponse(&DupDetectProto.ResponseCount); - job->SendReply(reply); - - return nullptr; - } - }; + //////////////////////////////////////////////////////////////////// + /// \brief DupDetect protocol that common between client and server + //////////////////////////////////////////////////////////////////// + /// \brief HostInfo request class + class THostInfoMessage: public TBusMessage { + public: + THostInfoMessage() + : TBusMessage(TYPE_HOSTINFOREQUEST) + { + } + THostInfoMessage(ECreateUninitialized) + : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) + { + } + + ~THostInfoMessage() override { + } + }; + + //////////////////////////////////////////////////////////////////// + /// \brief HostInfo reply class + class THostInfoReply: public TBusMessage { + public: + THostInfoReply() + : TBusMessage(TYPE_HOSTINFORESPONSE) + { + } + THostInfoReply(ECreateUninitialized) + : TBusMessage(MESSAGE_CREATE_UNINITIALIZED) + { + } + + ~THostInfoReply() override { + } + }; + + //////////////////////////////////////////////////////////////////// + /// \brief HostInfo protocol that common between client and server + class THostInfoProtocol: public TBusProtocol { + public: + THostInfoProtocol() + : TBusProtocol("HOSTINFO", 0) + { + } + /// serialized protocol specific data into TBusData + void Serialize(const TBusMessage* mess, TBuffer& data) override { + Y_UNUSED(data); + Y_UNUSED(mess); + } + + /// deserialized TBusData into new instance of the message + TAutoPtr<TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) override { + Y_UNUSED(payload); + + if (messageType == TYPE_HOSTINFOREQUEST) { + return new THostInfoMessage(MESSAGE_CREATE_UNINITIALIZED); + } else if (messageType == TYPE_HOSTINFORESPONSE) { + return new THostInfoReply(MESSAGE_CREATE_UNINITIALIZED); + } else { + Y_FAIL("unknown"); + } + } + }; + + ////////////////////////////////////////////////////////////// + /// \brief HostInfo handler (should convert it to module too) + struct THostInfoHandler: public TBusServerHandlerError { + TBusServerSessionPtr Session; + TBusServerSessionConfig HostInfoConfig; + THostInfoProtocol HostInfoProto; + + THostInfoHandler(TBusMessageQueue* queue) { + Session = TBusServerSession::Create(&HostInfoProto, this, HostInfoConfig, queue); + } + + void OnMessage(TOnMessageContext& mess) override { + usleep(10 * 1000); /// pretend we are doing something + + TAutoPtr<THostInfoReply> reply(new THostInfoReply()); + + mess.SendReplyMove(reply); + } + + TNetAddr GetActualListenAddr() { + return TNetAddr("localhost", Session->GetActualListenPort()); + } + }; + + ////////////////////////////////////////////////////////////// + /// \brief DupDetect handler (should convert it to module too) + struct TDupDetectHandler: public TBusClientHandlerError { + TNetAddr ServerAddr; + + TBusClientSessionPtr DupDetect; + TBusClientSessionConfig DupDetectConfig; + TExampleProtocol DupDetectProto; + + int NumMessages; + int NumReplies; + + TDupDetectHandler(const TNetAddr& serverAddr, TBusMessageQueuePtr queue) + : ServerAddr(serverAddr) + { + DupDetect = TBusClientSession::Create(&DupDetectProto, this, DupDetectConfig, queue); + DupDetect->RegisterService("localhost"); + } + + void Work() { + NumMessages = 10; + NumReplies = 0; + + for (int i = 0; i < NumMessages; i++) { + TExampleRequest* mess = new TExampleRequest(&DupDetectProto.RequestCount); + DupDetect->SendMessage(mess, &ServerAddr); + } + } + + void OnReply(TAutoPtr<TBusMessage> mess, TAutoPtr<TBusMessage> reply) override { + Y_UNUSED(mess); + Y_UNUSED(reply); + NumReplies++; + } + }; + + ///////////////////////////////////////////////////////////////// + /// \brief DupDetect module + + struct TDupDetectModule: public TBusModule { + TNetAddr HostInfoAddr; + + TBusClientSessionPtr HostInfoClientSession; + TBusClientSessionConfig HostInfoConfig; + THostInfoProtocol HostInfoProto; + + TExampleProtocol DupDetectProto; + TBusServerSessionConfig DupDetectConfig; + + TNetAddr ListenAddr; + + TDupDetectModule(const TNetAddr& hostInfoAddr) + : TBusModule("DUPDETECTMODULE") + , HostInfoAddr(hostInfoAddr) + { + } + + bool Init(TBusMessageQueue* queue) { + HostInfoClientSession = CreateDefaultSource(*queue, &HostInfoProto, HostInfoConfig); + HostInfoClientSession->RegisterService("localhost"); + + return TBusModule::CreatePrivateSessions(queue); + } + + TBusServerSessionPtr CreateExtSession(TBusMessageQueue& queue) override { + TBusServerSessionPtr session = CreateDefaultDestination(queue, &DupDetectProto, DupDetectConfig); + + ListenAddr = TNetAddr("localhost", session->GetActualListenPort()); + + return session; + } + + /// entry point into module, first function to call + TJobHandler Start(TBusJob* job, TBusMessage* mess) override { + TExampleRequest* dmess = dynamic_cast<TExampleRequest*>(mess); + Y_UNUSED(dmess); + + THostInfoMessage* hmess = new THostInfoMessage(); + + /// send message to imaginary hostinfo server + job->Send(hmess, HostInfoClientSession, TReplyHandler(), 0, HostInfoAddr); + + return TJobHandler(&TDupDetectModule::ProcessHostInfo); + } + + /// next handler is executed when all outstanding requests from previous handler is completed + TJobHandler ProcessHostInfo(TBusJob* job, TBusMessage* mess) { + TExampleRequest* dmess = dynamic_cast<TExampleRequest*>(mess); + Y_UNUSED(dmess); + + THostInfoMessage* hmess = job->Get<THostInfoMessage>(); + THostInfoReply* hreply = job->Get<THostInfoReply>(); + EMessageStatus hstatus = job->GetStatus<THostInfoMessage>(); + Y_ASSERT(hmess != nullptr); + Y_ASSERT(hreply != nullptr); + Y_ASSERT(hstatus == MESSAGE_OK); + + return TJobHandler(&TDupDetectModule::Finish); + } + + /// last handler sends reply and returns NULL + TJobHandler Finish(TBusJob* job, TBusMessage* mess) { + Y_UNUSED(mess); + + TExampleResponse* reply = new TExampleResponse(&DupDetectProto.ResponseCount); + job->SendReply(reply); + + return nullptr; + } + }; } -} +} diff --git a/library/cpp/messagebus/test/ut/one_way_ut.cpp b/library/cpp/messagebus/test/ut/one_way_ut.cpp index 9c21227e2b..a8e0cb960b 100644 --- a/library/cpp/messagebus/test/ut/one_way_ut.cpp +++ b/library/cpp/messagebus/test/ut/one_way_ut.cpp @@ -1,7 +1,7 @@ /////////////////////////////////////////////////////////////////// /// \file -/// \brief Example of reply-less communication - +/// \brief Example of reply-less communication + /// This example demostrates how asynchronous message passing library /// can be used to send message and do not wait for reply back. /// The usage of reply-less communication should be restricted to @@ -9,19 +9,19 @@ /// utility. Removing replies from the communication removes any restriction /// on how many message can be send to server and rougue clients may overwelm /// server without thoughtput control. - + /// 1) To implement reply-less client \n - -/// Call NBus::TBusSession::AckMessage() -/// from within NBus::IMessageHandler::OnSent() handler when message has -/// gone into wire on client end. See example in NBus::NullClient::OnMessageSent(). + +/// Call NBus::TBusSession::AckMessage() +/// from within NBus::IMessageHandler::OnSent() handler when message has +/// gone into wire on client end. See example in NBus::NullClient::OnMessageSent(). /// Discard identity for reply message. - + /// 2) To implement reply-less server \n - + /// Call NBus::TBusSession::AckMessage() from within NBus::IMessageHandler::OnMessage() -/// handler when message has been received on server end. -/// See example in NBus::NullServer::OnMessage(). +/// handler when message has been received on server end. +/// See example in NBus::NullServer::OnMessage(). /// Discard identity for reply message. #include <library/cpp/messagebus/test/helper/alloc_counter.h> @@ -41,7 +41,7 @@ using namespace NBus::NTest; //////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////// /// \brief Reply-less client and handler -struct NullClient : TBusClientHandlerError { +struct NullClient : TBusClientHandlerError { TNetAddr ServerAddr; TBusMessageQueuePtr Queue; @@ -69,11 +69,11 @@ struct NullClient : TBusClientHandlerError { } /// dispatch of requests is done here - void Work() { + void Work() { int batch = 10; - for (int i = 0; i < batch; i++) { - TExampleRequest* mess = new TExampleRequest(&Proto.RequestCount); + for (int i = 0; i < batch; i++) { + TExampleRequest* mess = new TExampleRequest(&Proto.RequestCount); mess->Data = "TADA"; Session->SendMessageOneWay(mess, &ServerAddr); } @@ -112,7 +112,7 @@ public: /// when message comes do not send reply, just acknowledge void OnMessage(TOnMessageContext& mess) override { - TExampleRequest* fmess = static_cast<TExampleRequest*>(mess.GetMessage()); + TExampleRequest* fmess = static_cast<TExampleRequest*>(mess.GetMessage()); Y_ASSERT(fmess->Data == "TADA"); @@ -126,7 +126,7 @@ public: void OnSent(TAutoPtr<TBusMessage> mess) override { Y_UNUSED(mess); Y_FAIL("This server does not sent replies"); - } + } }; Y_UNIT_TEST_SUITE(TMessageBusTests_OneWay) { @@ -158,8 +158,8 @@ Y_UNIT_TEST_SUITE(TMessageBusTests_OneWay) { TMessageTooLargeClient(unsigned port) : NullClient(TNetAddr("localhost", port), Config()) - { - } + { + } ~TMessageTooLargeClient() override { Session->Shutdown(); @@ -187,7 +187,7 @@ Y_UNIT_TEST_SUITE(TMessageBusTests_OneWay) { client.GotTooLarge.WaitI(); } - struct TCheckTimeoutClient: public NullClient { + struct TCheckTimeoutClient: public NullClient { ~TCheckTimeoutClient() override { Session->Shutdown(); } @@ -200,10 +200,10 @@ Y_UNIT_TEST_SUITE(TMessageBusTests_OneWay) { return sessionConfig; } - TCheckTimeoutClient(const TNetAddr& serverAddr) - : NullClient(serverAddr, SessionConfig()) - { - } + TCheckTimeoutClient(const TNetAddr& serverAddr) + : NullClient(serverAddr, SessionConfig()) + { + } TSystemEvent GotError; diff --git a/library/cpp/messagebus/test/ut/starter_ut.cpp b/library/cpp/messagebus/test/ut/starter_ut.cpp index dd4d3aaa5e..b9ff9a449d 100644 --- a/library/cpp/messagebus/test/ut/starter_ut.cpp +++ b/library/cpp/messagebus/test/ut/starter_ut.cpp @@ -8,7 +8,7 @@ using namespace NBus; using namespace NBus::NTest; Y_UNIT_TEST_SUITE(TBusStarterTest) { - struct TStartJobTestModule: public TExampleModule { + struct TStartJobTestModule: public TExampleModule { using TBusModule::CreateDefaultStarter; TAtomic StartCount; @@ -75,7 +75,7 @@ Y_UNIT_TEST_SUITE(TBusStarterTest) { module.Shutdown(); } - struct TSleepModule: public TExampleServerModule { + struct TSleepModule: public TExampleServerModule { TSystemEvent MessageReceivedEvent; TJobHandler Start(TBusJob* job, TBusMessage* mess) override { @@ -110,7 +110,7 @@ Y_UNIT_TEST_SUITE(TBusStarterTest) { module.Shutdown(); } - struct TSendReplyModule: public TExampleServerModule { + struct TSendReplyModule: public TExampleServerModule { TSystemEvent MessageReceivedEvent; TJobHandler Start(TBusJob* job, TBusMessage* mess) override { diff --git a/library/cpp/messagebus/test/ut/sync_client_ut.cpp b/library/cpp/messagebus/test/ut/sync_client_ut.cpp index 400128193f..7a7189dbec 100644 --- a/library/cpp/messagebus/test/ut/sync_client_ut.cpp +++ b/library/cpp/messagebus/test/ut/sync_client_ut.cpp @@ -2,67 +2,67 @@ #include <library/cpp/messagebus/test/helper/object_count_check.h> namespace NBus { - namespace NTest { - using namespace std; + namespace NTest { + using namespace std; - //////////////////////////////////////////////////////////////////// - /// \brief Client for sending synchronous message to local server - struct TSyncClient { - TNetAddr ServerAddr; + //////////////////////////////////////////////////////////////////// + /// \brief Client for sending synchronous message to local server + struct TSyncClient { + TNetAddr ServerAddr; - TExampleProtocol Proto; - TBusMessageQueuePtr Bus; - TBusSyncClientSessionPtr Session; + TExampleProtocol Proto; + TBusMessageQueuePtr Bus; + TBusSyncClientSessionPtr Session; - int NumReplies; - int NumMessages; + int NumReplies; + int NumMessages; - /// constructor creates instances of queue, protocol and session - TSyncClient(const TNetAddr& serverAddr) - : ServerAddr(serverAddr) - { - /// create or get instance of message queue, need one per application - Bus = CreateMessageQueue(); + /// constructor creates instances of queue, protocol and session + TSyncClient(const TNetAddr& serverAddr) + : ServerAddr(serverAddr) + { + /// create or get instance of message queue, need one per application + Bus = CreateMessageQueue(); - NumReplies = 0; - NumMessages = 10; + NumReplies = 0; + NumMessages = 10; - /// register source/client session - TBusClientSessionConfig sessionConfig; - Session = Bus->CreateSyncSource(&Proto, sessionConfig); - Session->RegisterService("localhost"); - } + /// register source/client session + TBusClientSessionConfig sessionConfig; + Session = Bus->CreateSyncSource(&Proto, sessionConfig); + Session->RegisterService("localhost"); + } - ~TSyncClient() { - Session->Shutdown(); - } + ~TSyncClient() { + Session->Shutdown(); + } - /// dispatch of requests is done here - void Work() { - for (int i = 0; i < NumMessages; i++) { - THolder<TExampleRequest> mess(new TExampleRequest(&Proto.RequestCount)); - EMessageStatus status; - THolder<TBusMessage> reply(Session->SendSyncMessage(mess.Get(), status, &ServerAddr)); - if (!!reply) { - NumReplies++; - } - } - } - }; + /// dispatch of requests is done here + void Work() { + for (int i = 0; i < NumMessages; i++) { + THolder<TExampleRequest> mess(new TExampleRequest(&Proto.RequestCount)); + EMessageStatus status; + THolder<TBusMessage> reply(Session->SendSyncMessage(mess.Get(), status, &ServerAddr)); + if (!!reply) { + NumReplies++; + } + } + } + }; Y_UNIT_TEST_SUITE(SyncClientTest) { Y_UNIT_TEST(TestSync) { - TObjectCountCheck objectCountCheck; + TObjectCountCheck objectCountCheck; - TExampleServer server; - TSyncClient client(server.GetActualListenAddr()); - client.Work(); - // assert correct number of replies - UNIT_ASSERT_EQUAL(client.NumReplies, client.NumMessages); - // assert that there is no message left in flight - UNIT_ASSERT_EQUAL(server.Session->GetInFlight(), 0); - UNIT_ASSERT_EQUAL(client.Session->GetInFlight(), 0); - } + TExampleServer server; + TSyncClient client(server.GetActualListenAddr()); + client.Work(); + // assert correct number of replies + UNIT_ASSERT_EQUAL(client.NumReplies, client.NumMessages); + // assert that there is no message left in flight + UNIT_ASSERT_EQUAL(server.Session->GetInFlight(), 0); + UNIT_ASSERT_EQUAL(client.Session->GetInFlight(), 0); + } } } diff --git a/library/cpp/messagebus/test/ya.make b/library/cpp/messagebus/test/ya.make index 0dc4bd4720..5a3b771a1c 100644 --- a/library/cpp/messagebus/test/ya.make +++ b/library/cpp/messagebus/test/ya.make @@ -1,7 +1,7 @@ OWNER(g:messagebus) -RECURSE( +RECURSE( example perftest ut -) +) diff --git a/library/cpp/messagebus/test_utils.h b/library/cpp/messagebus/test_utils.h index 2abdf504b1..5a6724c27a 100644 --- a/library/cpp/messagebus/test_utils.h +++ b/library/cpp/messagebus/test_utils.h @@ -1,12 +1,12 @@ #pragma once // Do nothing if there is no support for IPv4 -#define ASSUME_IP_V4_ENABLED \ - do { \ - try { \ - TNetworkAddress("192.168.0.42", 80); \ - } catch (const TNetworkResolutionError& ex) { \ - Y_UNUSED(ex); \ - return; \ - } \ +#define ASSUME_IP_V4_ENABLED \ + do { \ + try { \ + TNetworkAddress("192.168.0.42", 80); \ + } catch (const TNetworkResolutionError& ex) { \ + Y_UNUSED(ex); \ + return; \ + } \ } while (0) diff --git a/library/cpp/messagebus/use_after_free_checker.cpp b/library/cpp/messagebus/use_after_free_checker.cpp index 4904e7c614..01f38d8d34 100644 --- a/library/cpp/messagebus/use_after_free_checker.cpp +++ b/library/cpp/messagebus/use_after_free_checker.cpp @@ -3,8 +3,8 @@ #include <util/system/yassert.h> namespace { - const ui64 VALID = (ui64)0xAABBCCDDEEFF0011LL; - const ui64 INVALID = (ui64)0x1122334455667788LL; + const ui64 VALID = (ui64)0xAABBCCDDEEFF0011LL; + const ui64 INVALID = (ui64)0x1122334455667788LL; } TUseAfterFreeChecker::TUseAfterFreeChecker() @@ -12,7 +12,7 @@ TUseAfterFreeChecker::TUseAfterFreeChecker() { } -TUseAfterFreeChecker::~TUseAfterFreeChecker() { +TUseAfterFreeChecker::~TUseAfterFreeChecker() { Y_VERIFY(Magic == VALID, "Corrupted"); Magic = INVALID; } diff --git a/library/cpp/messagebus/use_after_free_checker.h b/library/cpp/messagebus/use_after_free_checker.h index 590b076156..7f6db63af7 100644 --- a/library/cpp/messagebus/use_after_free_checker.h +++ b/library/cpp/messagebus/use_after_free_checker.h @@ -6,7 +6,7 @@ class TUseAfterFreeChecker { private: ui64 Magic; - + public: TUseAfterFreeChecker(); ~TUseAfterFreeChecker(); @@ -17,7 +17,7 @@ public: class TUseAfterFreeCheckerGuard { private: const TUseAfterFreeChecker& Check; - + public: TUseAfterFreeCheckerGuard(const TUseAfterFreeChecker& check) : Check(check) diff --git a/library/cpp/messagebus/use_count_checker.cpp b/library/cpp/messagebus/use_count_checker.cpp index c6243ea21f..387688af39 100644 --- a/library/cpp/messagebus/use_count_checker.cpp +++ b/library/cpp/messagebus/use_count_checker.cpp @@ -8,7 +8,7 @@ TUseCountChecker::TUseCountChecker() { TUseCountChecker::~TUseCountChecker() { TAtomicBase count = Counter.Val(); - Y_VERIFY(count == 0, "must not release when count is not zero: %ld", (long)count); + Y_VERIFY(count == 0, "must not release when count is not zero: %ld", (long)count); } void TUseCountChecker::Inc() { @@ -21,8 +21,8 @@ void TUseCountChecker::Dec() { TUseCountHolder::TUseCountHolder() : CurrentChecker(nullptr) -{ -} +{ +} TUseCountHolder::TUseCountHolder(TUseCountChecker* currentChecker) : CurrentChecker(currentChecker) @@ -32,7 +32,7 @@ TUseCountHolder::TUseCountHolder(TUseCountChecker* currentChecker) } } -TUseCountHolder::~TUseCountHolder() { +TUseCountHolder::~TUseCountHolder() { if (!!CurrentChecker) { CurrentChecker->Dec(); } diff --git a/library/cpp/messagebus/use_count_checker.h b/library/cpp/messagebus/use_count_checker.h index 70bef6fa8a..9e008f0c22 100644 --- a/library/cpp/messagebus/use_count_checker.h +++ b/library/cpp/messagebus/use_count_checker.h @@ -5,7 +5,7 @@ class TUseCountChecker { private: TAtomicCounter Counter; - + public: TUseCountChecker(); ~TUseCountChecker(); @@ -16,7 +16,7 @@ public: class TUseCountHolder { private: TUseCountChecker* CurrentChecker; - + public: TUseCountHolder(); explicit TUseCountHolder(TUseCountChecker* currentChecker); diff --git a/library/cpp/messagebus/vector_swaps.h b/library/cpp/messagebus/vector_swaps.h index b920bcf03e..7f22fd7eb3 100644 --- a/library/cpp/messagebus/vector_swaps.h +++ b/library/cpp/messagebus/vector_swaps.h @@ -8,7 +8,7 @@ #include <stdlib.h> template <typename T, class A = std::allocator<T>> -class TVectorSwaps : TNonCopyable { +class TVectorSwaps : TNonCopyable { private: T* Start; T* Finish; @@ -23,15 +23,15 @@ public: typedef T* iterator; typedef const T* const_iterator; - typedef std::reverse_iterator<iterator> reverse_iterator; - typedef std::reverse_iterator<const_iterator> const_reverse_iterator; + typedef std::reverse_iterator<iterator> reverse_iterator; + typedef std::reverse_iterator<const_iterator> const_reverse_iterator; - TVectorSwaps() - : Start() - , Finish() - , EndOfStorage() - { - } + TVectorSwaps() + : Start() + , Finish() + , EndOfStorage() + { + } ~TVectorSwaps() { for (size_t i = 0; i < size(); ++i) { @@ -94,19 +94,19 @@ public: return Finish; } - reverse_iterator rbegin() { - return reverse_iterator(end()); - } - reverse_iterator rend() { - return reverse_iterator(begin()); - } + reverse_iterator rbegin() { + return reverse_iterator(end()); + } + reverse_iterator rend() { + return reverse_iterator(begin()); + } - const_reverse_iterator rbegin() const { - return reverse_iterator(end()); - } - const_reverse_iterator rend() const { - return reverse_iterator(begin()); - } + const_reverse_iterator rbegin() const { + return reverse_iterator(end()); + } + const_reverse_iterator rend() const { + return reverse_iterator(begin()); + } void swap(TVectorSwaps<T>& that) { DoSwap(Start, that.Start); @@ -121,7 +121,7 @@ public: size_t newCapacity = FastClp2(n); TVectorSwaps<T> tmp; - tmp.Start = (T*)malloc(sizeof(T) * newCapacity); + tmp.Start = (T*)malloc(sizeof(T) * newCapacity); Y_VERIFY(!!tmp.Start); tmp.EndOfStorage = tmp.Start + newCapacity; diff --git a/library/cpp/messagebus/vector_swaps_ut.cpp b/library/cpp/messagebus/vector_swaps_ut.cpp index 693cc6857b..788cccc7e6 100644 --- a/library/cpp/messagebus/vector_swaps_ut.cpp +++ b/library/cpp/messagebus/vector_swaps_ut.cpp @@ -4,7 +4,7 @@ Y_UNIT_TEST_SUITE(TVectorSwapsTest) { Y_UNIT_TEST(Simple) { - TVectorSwaps<THolder<unsigned>> v; + TVectorSwaps<THolder<unsigned>> v; for (unsigned i = 0; i < 100; ++i) { THolder<unsigned> tmp(new unsigned(i)); v.push_back(tmp); diff --git a/library/cpp/messagebus/www/concat_strings.h b/library/cpp/messagebus/www/concat_strings.h index 7b730564eb..77a77d4695 100644 --- a/library/cpp/messagebus/www/concat_strings.h +++ b/library/cpp/messagebus/www/concat_strings.h @@ -5,16 +5,16 @@ // ATTN: not equivalent to TString::Join - cat concat anything "outputable" to stream, not only TString convertable types. -inline void DoConcatStrings(TStringStream&) { +inline void DoConcatStrings(TStringStream&) { } -template <class T, class... R> -inline void DoConcatStrings(TStringStream& ss, const T& t, const R&... r) { +template <class T, class... R> +inline void DoConcatStrings(TStringStream& ss, const T& t, const R&... r) { ss << t; DoConcatStrings(ss, r...); } -template <class... R> +template <class... R> inline TString ConcatStrings(const R&... r) { TStringStream ss; DoConcatStrings(ss, r...); diff --git a/library/cpp/messagebus/www/html_output.cpp b/library/cpp/messagebus/www/html_output.cpp index 10ea2e163b..5128f74718 100644 --- a/library/cpp/messagebus/www/html_output.cpp +++ b/library/cpp/messagebus/www/html_output.cpp @@ -1,4 +1,4 @@ #include "html_output.h" -Y_POD_THREAD(IOutputStream*) -HtmlOutputStreamPtr; +Y_POD_THREAD(IOutputStream*) +HtmlOutputStreamPtr; diff --git a/library/cpp/messagebus/www/html_output.h b/library/cpp/messagebus/www/html_output.h index 27e77adefa..50c3f1fa15 100644 --- a/library/cpp/messagebus/www/html_output.h +++ b/library/cpp/messagebus/www/html_output.h @@ -32,26 +32,26 @@ struct TChars { TString Text; bool NeedEscape; - TChars(TStringBuf text) - : Text(text) - , NeedEscape(true) - { - } - TChars(TStringBuf text, bool escape) - : Text(text) - , NeedEscape(escape) - { - } - TChars(const char* text) - : Text(text) - , NeedEscape(true) - { - } - TChars(const char* text, bool escape) - : Text(text) - , NeedEscape(escape) - { - } + TChars(TStringBuf text) + : Text(text) + , NeedEscape(true) + { + } + TChars(TStringBuf text, bool escape) + : Text(text) + , NeedEscape(escape) + { + } + TChars(const char* text) + : Text(text) + , NeedEscape(true) + { + } + TChars(const char* text, bool escape) + : Text(text) + , NeedEscape(escape) + { + } TString Escape() { if (NeedEscape) { @@ -67,13 +67,13 @@ struct TAttr { TString Value; TAttr(TStringBuf name, TStringBuf value) - : Name(name) - , Value(value) - { - } + : Name(name) + , Value(value) + { + } - TAttr() { - } + TAttr() { + } bool operator!() const { return !Name; @@ -239,9 +239,9 @@ static inline void BootstrapInfo(TStringBuf text) { static inline void ScriptHref(TStringBuf href) { Open("script", - TAttr("language", "javascript"), - TAttr("type", "text/javascript"), - TAttr("src", href)); + TAttr("language", "javascript"), + TAttr("type", "text/javascript"), + TAttr("src", href)); Close("script"); Nl(); } @@ -290,17 +290,17 @@ struct TTagGuard { struct TDivGuard: public TTagGuard { TDivGuard(TStringBuf cssClass, TStringBuf id = "") : TTagGuard("div", cssClass, id) - { - } + { + } TDivGuard(TAttr a1 = TAttr(), TAttr a2 = TAttr(), TAttr a3 = TAttr()) : TTagGuard("div", a1, a2, a3) - { - } + { + } }; struct TAGuard { - TAGuard(TStringBuf href) { + TAGuard(TStringBuf href) { OpenA(href); } @@ -318,7 +318,7 @@ struct TScriptFunctionGuard { Line("$(function() {"); } - ~TScriptFunctionGuard() { + ~TScriptFunctionGuard() { Line("});"); } }; diff --git a/library/cpp/messagebus/www/www.cpp b/library/cpp/messagebus/www/www.cpp index 62ec241d85..9af596a143 100644 --- a/library/cpp/messagebus/www/www.cpp +++ b/library/cpp/messagebus/www/www.cpp @@ -35,7 +35,7 @@ namespace { template <typename TValuePtr> struct TNamedValues { - TVector<std::pair<TString, TValuePtr>> Entries; + TVector<std::pair<TString, TValuePtr>> Entries; TValuePtr FindByName(TStringBuf name) { Y_VERIFY(!!name); @@ -79,7 +79,7 @@ namespace { } } - for (unsigned i = 1;; ++i) { + for (unsigned i = 1;; ++i) { TString prefix = p->GetNameInternal(); if (!prefix) { prefix = "unnamed"; @@ -169,7 +169,7 @@ const unsigned char WWW_STATIC_DATA[] = { #include "www_static.inc" }; -class TWwwStaticLoader: public TArchiveReader { +class TWwwStaticLoader: public TArchiveReader { public: TWwwStaticLoader() : TArchiveReader(TBlob::NoCopy(WWW_STATIC_DATA, sizeof(WWW_STATIC_DATA))) @@ -180,8 +180,8 @@ public: struct TBusWww::TImpl { // TODO: use weak pointers TNamedValues<TBusMessageQueuePtr> Queues; - TSessionValues<TIntrusivePtr<TBusClientSession>> ClientSessions; - TSessionValues<TIntrusivePtr<TBusServerSession>> ServerSessions; + TSessionValues<TIntrusivePtr<TBusClientSession>> ClientSessions; + TSessionValues<TIntrusivePtr<TBusServerSession>> ServerSessions; TSessionValues<TBusModuleInternalPtr> Modules; TMutex Mutex; @@ -250,12 +250,12 @@ struct TBusWww::TImpl { const TOptionalParams& Params; TRequest(TImpl* outer, IOutputStream& os, const TCgiParameters& cgiParams, const TOptionalParams& params) - : Outer(outer) - , Os(os) - , CgiParams(cgiParams) - , Params(params) - { - } + : Outer(outer) + , Os(os) + , CgiParams(cgiParams) + , Params(params) + { + } void CrumbsParentLinks() { for (unsigned i = 0; i < Params.ParentLinks.size(); ++i) { @@ -482,7 +482,7 @@ struct TBusWww::TImpl { if (needTick) { ticks.BeginList(); ticks.WriteInt(i); - ticks.WriteString(Sprintf(":%02u:%02u", (unsigned)minuteOfHour, (unsigned)secondOfMinute)); + ticks.WriteString(Sprintf(":%02u:%02u", (unsigned)minuteOfHour, (unsigned)secondOfMinute)); ticks.EndList(); } } @@ -554,10 +554,10 @@ struct TBusWww::TImpl { { TDivGuard div; TTagGuard button("button", - TAttr("type", "button"), - TAttr("class", "btn"), - TAttr("data-toggle", "collapse"), - TAttr("data-target", "#connections")); + TAttr("type", "button"), + TAttr("class", "btn"), + TAttr("data-toggle", "collapse"), + TAttr("data-target", "#connections")); Text("Show connection details"); } { @@ -572,7 +572,7 @@ struct TBusWww::TImpl { H3("Message process time histogram"); const TDurationHistogram& h = - dumpStatus.ConnectionStatusSummary.WriterStatus.Incremental.ProcessDurationHistogram; + dumpStatus.ConnectionStatusSummary.WriterStatus.Incremental.ProcessDurationHistogram; { TDivGuard div; @@ -630,7 +630,7 @@ struct TBusWww::TImpl { } void WriteMessageCounterSensors(NMonitoring::TDeprecatedJsonWriter& sj, - TStringBuf labelName, TStringBuf sessionName, bool read, const TMessageCounter& counter) { + TStringBuf labelName, TStringBuf sessionName, bool read, const TMessageCounter& counter) { TStringBuf readOrWrite = read ? "read" : "write"; sj.OpenMetric(); @@ -647,7 +647,7 @@ struct TBusWww::TImpl { } void WriteSessionStatus(NMonitoring::TDeprecatedJsonWriter& sj, TStringBuf sessionName, bool client, - TBusSession* session) { + TBusSession* session) { TStringBuf labelName = client ? "mb_client_session" : "mb_server_session"; auto status = session->GetStatusRecordInternal(); @@ -675,9 +675,9 @@ struct TBusWww::TImpl { } WriteMessageCounterSensors(sj, labelName, sessionName, false, - status.ConnectionStatusSummary.WriterStatus.Incremental.MessageCounter); + status.ConnectionStatusSummary.WriterStatus.Incremental.MessageCounter); WriteMessageCounterSensors(sj, labelName, sessionName, true, - status.ConnectionStatusSummary.ReaderStatus.Incremental.MessageCounter); + status.ConnectionStatusSummary.ReaderStatus.Incremental.MessageCounter); } void ServeSolomonJson(const TString& q, const TString& cs, const TString& ss) { @@ -813,14 +813,14 @@ NBus::TBusWww::TBusWww() { } -NBus::TBusWww::~TBusWww() { +NBus::TBusWww::~TBusWww() { } -void NBus::TBusWww::RegisterClientSession(TBusClientSessionPtr s) { +void NBus::TBusWww::RegisterClientSession(TBusClientSessionPtr s) { Impl->RegisterClientSession(s); } -void TBusWww::RegisterServerSession(TBusServerSessionPtr s) { +void TBusWww::RegisterServerSession(TBusServerSessionPtr s) { Impl->RegisterServerSession(s); } @@ -833,8 +833,8 @@ void TBusWww::RegisterModule(TBusModule* module) { } void TBusWww::ServeHttp(IOutputStream& httpOutputStream, - const TCgiParameters& queryArgs, - const TBusWww::TOptionalParams& params) { + const TCgiParameters& queryArgs, + const TBusWww::TOptionalParams& params) { Impl->ServeHttp(httpOutputStream, queryArgs, params); } @@ -861,8 +861,8 @@ struct TBusWwwHttpServer::TImpl: public THttpServer::ICallBack { TClientRequestImpl(TBusWwwHttpServer::TImpl* outer) : Outer(outer) - { - } + { + } bool Reply(void*) override { Outer->ServeRequest(Input(), Output()); @@ -878,8 +878,8 @@ struct TBusWwwHttpServer::TImpl: public THttpServer::ICallBack { } TStringStream ss; ss << "HTTP/1.1 " - << code << " " << text << "\r\nConnection: Close\r\n\r\n" - << content; + << code << " " << text << "\r\nConnection: Close\r\n\r\n" + << content; return ss.Str(); } @@ -908,7 +908,7 @@ struct TBusWwwHttpServer::TImpl: public THttpServer::ICallBack { Www->ServeHttp(output, cgiParams, params); } catch (...) { output << MakeSimpleResponse(500, "Exception", - TString() + "Exception: " + CurrentExceptionMessage()); + TString() + "Exception: " + CurrentExceptionMessage()); } } @@ -926,5 +926,5 @@ NBus::TBusWwwHttpServer::TBusWwwHttpServer(TIntrusivePtr<TBusWww> www, unsigned { } -NBus::TBusWwwHttpServer::~TBusWwwHttpServer() { +NBus::TBusWwwHttpServer::~TBusWwwHttpServer() { } diff --git a/library/cpp/messagebus/www/www.h b/library/cpp/messagebus/www/www.h index 6cd652b477..064f42b9db 100644 --- a/library/cpp/messagebus/www/www.h +++ b/library/cpp/messagebus/www/www.h @@ -8,38 +8,38 @@ #include <library/cpp/cgiparam/cgiparam.h> namespace NBus { - class TBusWww: public TAtomicRefCount<TBusWww> { - public: - struct TLink { - TString Title; - TString Href; - }; + class TBusWww: public TAtomicRefCount<TBusWww> { + public: + struct TLink { + TString Title; + TString Href; + }; - struct TOptionalParams { - TVector<TLink> ParentLinks; - }; + struct TOptionalParams { + TVector<TLink> ParentLinks; + }; - TBusWww(); - ~TBusWww(); + TBusWww(); + ~TBusWww(); - void RegisterClientSession(TBusClientSessionPtr); - void RegisterServerSession(TBusServerSessionPtr); - void RegisterQueue(TBusMessageQueuePtr); - void RegisterModule(TBusModule*); + void RegisterClientSession(TBusClientSessionPtr); + void RegisterServerSession(TBusServerSessionPtr); + void RegisterQueue(TBusMessageQueuePtr); + void RegisterModule(TBusModule*); - void ServeHttp(IOutputStream& httpOutputStream, const TCgiParameters& queryArgs, const TOptionalParams& params = TOptionalParams()); + void ServeHttp(IOutputStream& httpOutputStream, const TCgiParameters& queryArgs, const TOptionalParams& params = TOptionalParams()); - struct TImpl; - THolder<TImpl> Impl; - }; + struct TImpl; + THolder<TImpl> Impl; + }; - class TBusWwwHttpServer { - public: - TBusWwwHttpServer(TIntrusivePtr<TBusWww> www, unsigned port); - ~TBusWwwHttpServer(); + class TBusWwwHttpServer { + public: + TBusWwwHttpServer(TIntrusivePtr<TBusWww> www, unsigned port); + ~TBusWwwHttpServer(); - struct TImpl; - THolder<TImpl> Impl; - }; + struct TImpl; + THolder<TImpl> Impl; + }; -} +} diff --git a/library/cpp/messagebus/ybus.h b/library/cpp/messagebus/ybus.h index de21ad8521..784fba4c1b 100644 --- a/library/cpp/messagebus/ybus.h +++ b/library/cpp/messagebus/ybus.h @@ -1,8 +1,8 @@ -#pragma once - +#pragma once + /// Asynchronous Messaging Library implements framework for sending and /// receiving messages between loosely connected processes. - + #include "coreconn.h" #include "defs.h" #include "handler.h" @@ -35,171 +35,171 @@ #include <util/system/mutex.h> namespace NBus { - //////////////////////////////////////////////////////// - /// \brief Common structure to store address information - - int CompareByHost(const IRemoteAddr& l, const IRemoteAddr& r) noexcept; - bool operator<(const TNetAddr& a1, const TNetAddr& a2); // compare by addresses - - ///////////////////////////////////////////////////////////////////////// - /// \brief Handles routing and data encoding to/from wire - - /// Protocol is stateless threadsafe singleton object that - /// encapsulates relationship between a message (TBusMessage) object - /// and destination server. Protocol object is reponsible for serializing in-memory - /// message and reply into the wire, retuning name of the service and resource - /// distribution key for given protocol. - - /// Protocol object should transparently handle messages and replies. - /// This is interface only class, actuall instances of the protocols - /// should be created using templates inhereted from this base class. - class TBusProtocol { - private: - TString ServiceName; - int ServicePort; - - public: - TBusProtocol(TBusService name = "UNKNOWN", int port = 0) - : ServiceName(name) - , ServicePort(port) - { - } - - /// returns service type for this protocol and message - TBusService GetService() const { + //////////////////////////////////////////////////////// + /// \brief Common structure to store address information + + int CompareByHost(const IRemoteAddr& l, const IRemoteAddr& r) noexcept; + bool operator<(const TNetAddr& a1, const TNetAddr& a2); // compare by addresses + + ///////////////////////////////////////////////////////////////////////// + /// \brief Handles routing and data encoding to/from wire + + /// Protocol is stateless threadsafe singleton object that + /// encapsulates relationship between a message (TBusMessage) object + /// and destination server. Protocol object is reponsible for serializing in-memory + /// message and reply into the wire, retuning name of the service and resource + /// distribution key for given protocol. + + /// Protocol object should transparently handle messages and replies. + /// This is interface only class, actuall instances of the protocols + /// should be created using templates inhereted from this base class. + class TBusProtocol { + private: + TString ServiceName; + int ServicePort; + + public: + TBusProtocol(TBusService name = "UNKNOWN", int port = 0) + : ServiceName(name) + , ServicePort(port) + { + } + + /// returns service type for this protocol and message + TBusService GetService() const { return ServiceName.data(); - } + } - /// returns port number for destination session to open socket - int GetPort() const { - return ServicePort; - } + /// returns port number for destination session to open socket + int GetPort() const { + return ServicePort; + } - virtual ~TBusProtocol() { - } + virtual ~TBusProtocol() { + } - /// \brief serialized protocol specific data into TBusData - /// \note buffer passed to the function (data) is not empty, use append functions - virtual void Serialize(const TBusMessage* mess, TBuffer& data) = 0; + /// \brief serialized protocol specific data into TBusData + /// \note buffer passed to the function (data) is not empty, use append functions + virtual void Serialize(const TBusMessage* mess, TBuffer& data) = 0; - /// deserialized TBusData into new instance of the message - virtual TAutoPtr<TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) = 0; + /// deserialized TBusData into new instance of the message + virtual TAutoPtr<TBusMessage> Deserialize(ui16 messageType, TArrayRef<const char> payload) = 0; - /// returns key for messages of this protocol - virtual TBusKey GetKey(const TBusMessage*) { - return YBUS_KEYMIN; - } + /// returns key for messages of this protocol + virtual TBusKey GetKey(const TBusMessage*) { + return YBUS_KEYMIN; + } - /// default implementation of routing policy to allow overrides - virtual EMessageStatus GetDestination(const TBusClientSession* session, TBusMessage* mess, TBusLocator* locator, TNetAddr* addr); + /// default implementation of routing policy to allow overrides + virtual EMessageStatus GetDestination(const TBusClientSession* session, TBusMessage* mess, TBusLocator* locator, TNetAddr* addr); - /// codec for transport level compression - virtual NCodecs::TCodecPtr GetTransportCodec(void) const { - return NCodecs::ICodec::GetInstance("snappy"); - } - }; + /// codec for transport level compression + virtual NCodecs::TCodecPtr GetTransportCodec(void) const { + return NCodecs::ICodec::GetInstance("snappy"); + } + }; - class TBusSyncSourceSession: public TAtomicRefCount<TBusSyncSourceSession> { - friend class TBusMessageQueue; + class TBusSyncSourceSession: public TAtomicRefCount<TBusSyncSourceSession> { + friend class TBusMessageQueue; - public: - TBusSyncSourceSession(TIntrusivePtr< ::NBus::NPrivate::TBusSyncSourceSessionImpl> session); - ~TBusSyncSourceSession(); + public: + TBusSyncSourceSession(TIntrusivePtr< ::NBus::NPrivate::TBusSyncSourceSessionImpl> session); + ~TBusSyncSourceSession(); - void Shutdown(); + void Shutdown(); - TBusMessage* SendSyncMessage(TBusMessage* pMessage, EMessageStatus& status, const TNetAddr* addr = nullptr); + TBusMessage* SendSyncMessage(TBusMessage* pMessage, EMessageStatus& status, const TNetAddr* addr = nullptr); - int RegisterService(const char* hostname, TBusKey start = YBUS_KEYMIN, TBusKey end = YBUS_KEYMAX, EIpVersion ipVersion = EIP_VERSION_4); + int RegisterService(const char* hostname, TBusKey start = YBUS_KEYMIN, TBusKey end = YBUS_KEYMAX, EIpVersion ipVersion = EIP_VERSION_4); - int GetInFlight(); + int GetInFlight(); - const TBusProtocol* GetProto() const; + const TBusProtocol* GetProto() const; - const TBusClientSession* GetBusClientSessionWorkaroundDoNotUse() const; // It's for TLoadBalancedProtocol::GetDestination() function that really needs TBusClientSession* unlike all other protocols. Look at review 32425 (http://rb.yandex-team.ru/arc/r/32425/) for more information. - private: - TIntrusivePtr< ::NBus::NPrivate::TBusSyncSourceSessionImpl> Session; - }; + const TBusClientSession* GetBusClientSessionWorkaroundDoNotUse() const; // It's for TLoadBalancedProtocol::GetDestination() function that really needs TBusClientSession* unlike all other protocols. Look at review 32425 (http://rb.yandex-team.ru/arc/r/32425/) for more information. + private: + TIntrusivePtr< ::NBus::NPrivate::TBusSyncSourceSessionImpl> Session; + }; - using TBusSyncClientSessionPtr = TIntrusivePtr<TBusSyncSourceSession>; + using TBusSyncClientSessionPtr = TIntrusivePtr<TBusSyncSourceSession>; - /////////////////////////////////////////////////////////////////// - /// \brief Main message queue object, need one per application - class TBusMessageQueue: public TAtomicRefCount<TBusMessageQueue> { - /// allow mesage queue to be created only via factory - friend TBusMessageQueuePtr CreateMessageQueue(const TBusQueueConfig& config, NActor::TExecutorPtr executor, TBusLocator* locator, const char* name); - friend class ::NBus::NPrivate::TRemoteConnection; - friend struct ::NBus::NPrivate::TBusSessionImpl; - friend class ::NBus::NPrivate::TAcceptor; - friend struct ::NBus::TBusServerSession; + /////////////////////////////////////////////////////////////////// + /// \brief Main message queue object, need one per application + class TBusMessageQueue: public TAtomicRefCount<TBusMessageQueue> { + /// allow mesage queue to be created only via factory + friend TBusMessageQueuePtr CreateMessageQueue(const TBusQueueConfig& config, NActor::TExecutorPtr executor, TBusLocator* locator, const char* name); + friend class ::NBus::NPrivate::TRemoteConnection; + friend struct ::NBus::NPrivate::TBusSessionImpl; + friend class ::NBus::NPrivate::TAcceptor; + friend struct ::NBus::TBusServerSession; - private: - const TBusQueueConfig Config; - TMutex Lock; - TList<TIntrusivePtr< ::NBus::NPrivate::TBusSessionImpl>> Sessions; - TSimpleIntrusivePtr<TBusLocator> Locator; - NPrivate::TScheduler Scheduler; + private: + const TBusQueueConfig Config; + TMutex Lock; + TList<TIntrusivePtr< ::NBus::NPrivate::TBusSessionImpl>> Sessions; + TSimpleIntrusivePtr<TBusLocator> Locator; + NPrivate::TScheduler Scheduler; - ::NActor::TExecutorPtr WorkQueue; + ::NActor::TExecutorPtr WorkQueue; - TAtomic Running; + TAtomic Running; TSystemEvent ShutdownComplete; - private: - /// constructor is protected, used NBus::CreateMessageQueue() to create a instance - TBusMessageQueue(const TBusQueueConfig& config, NActor::TExecutorPtr executor, TBusLocator* locator, const char* name); + private: + /// constructor is protected, used NBus::CreateMessageQueue() to create a instance + TBusMessageQueue(const TBusQueueConfig& config, NActor::TExecutorPtr executor, TBusLocator* locator, const char* name); - public: - TString GetNameInternal() const; + public: + TString GetNameInternal() const; - ~TBusMessageQueue(); + ~TBusMessageQueue(); - void Stop(); - bool IsRunning(); + void Stop(); + bool IsRunning(); - public: - void EnqueueWork(TArrayRef< ::NActor::IWorkItem* const> w) { - WorkQueue->EnqueueWork(w); - } + public: + void EnqueueWork(TArrayRef< ::NActor::IWorkItem* const> w) { + WorkQueue->EnqueueWork(w); + } - ::NActor::TExecutor* GetExecutor() { - return WorkQueue.Get(); - } + ::NActor::TExecutor* GetExecutor() { + return WorkQueue.Get(); + } - TString GetStatus(ui16 flags = YBUS_STATUS_CONNS) const; - // without sessions - NPrivate::TBusMessageQueueStatus GetStatusRecordInternal() const; - TString GetStatusSelf() const; - TString GetStatusSingleLine() const; + TString GetStatus(ui16 flags = YBUS_STATUS_CONNS) const; + // without sessions + NPrivate::TBusMessageQueueStatus GetStatusRecordInternal() const; + TString GetStatusSelf() const; + TString GetStatusSingleLine() const; - TBusLocator* GetLocator() const { - return Locator.Get(); - } + TBusLocator* GetLocator() const { + return Locator.Get(); + } - TBusClientSessionPtr CreateSource(TBusProtocol* proto, IBusClientHandler* handler, const TBusClientSessionConfig& config, const TString& name = ""); - TBusSyncClientSessionPtr CreateSyncSource(TBusProtocol* proto, const TBusClientSessionConfig& config, bool needReply = true, const TString& name = ""); - TBusServerSessionPtr CreateDestination(TBusProtocol* proto, IBusServerHandler* hander, const TBusServerSessionConfig& config, const TString& name = ""); - TBusServerSessionPtr CreateDestination(TBusProtocol* proto, IBusServerHandler* hander, const TBusServerSessionConfig& config, const TVector<TBindResult>& bindTo, const TString& name = ""); + TBusClientSessionPtr CreateSource(TBusProtocol* proto, IBusClientHandler* handler, const TBusClientSessionConfig& config, const TString& name = ""); + TBusSyncClientSessionPtr CreateSyncSource(TBusProtocol* proto, const TBusClientSessionConfig& config, bool needReply = true, const TString& name = ""); + TBusServerSessionPtr CreateDestination(TBusProtocol* proto, IBusServerHandler* hander, const TBusServerSessionConfig& config, const TString& name = ""); + TBusServerSessionPtr CreateDestination(TBusProtocol* proto, IBusServerHandler* hander, const TBusServerSessionConfig& config, const TVector<TBindResult>& bindTo, const TString& name = ""); - private: - void Destroy(TBusSession* session); - void Destroy(TBusSyncClientSessionPtr session); + private: + void Destroy(TBusSession* session); + void Destroy(TBusSyncClientSessionPtr session); - public: - void Schedule(NPrivate::IScheduleItemAutoPtr i); + public: + void Schedule(NPrivate::IScheduleItemAutoPtr i); - private: - void DestroyAllSessions(); - void Add(TIntrusivePtr< ::NBus::NPrivate::TBusSessionImpl> session); - void Remove(TBusSession* session); - }; + private: + void DestroyAllSessions(); + void Add(TIntrusivePtr< ::NBus::NPrivate::TBusSessionImpl> session); + void Remove(TBusSession* session); + }; - ///////////////////////////////////////////////////////////////// - /// Factory methods to construct message queue - TBusMessageQueuePtr CreateMessageQueue(const char* name = ""); - TBusMessageQueuePtr CreateMessageQueue(NActor::TExecutorPtr executor, const char* name = ""); - TBusMessageQueuePtr CreateMessageQueue(const TBusQueueConfig& config, const char* name = ""); - TBusMessageQueuePtr CreateMessageQueue(const TBusQueueConfig& config, TBusLocator* locator, const char* name = ""); - TBusMessageQueuePtr CreateMessageQueue(const TBusQueueConfig& config, NActor::TExecutorPtr executor, TBusLocator* locator, const char* name = ""); + ///////////////////////////////////////////////////////////////// + /// Factory methods to construct message queue + TBusMessageQueuePtr CreateMessageQueue(const char* name = ""); + TBusMessageQueuePtr CreateMessageQueue(NActor::TExecutorPtr executor, const char* name = ""); + TBusMessageQueuePtr CreateMessageQueue(const TBusQueueConfig& config, const char* name = ""); + TBusMessageQueuePtr CreateMessageQueue(const TBusQueueConfig& config, TBusLocator* locator, const char* name = ""); + TBusMessageQueuePtr CreateMessageQueue(const TBusQueueConfig& config, NActor::TExecutorPtr executor, TBusLocator* locator, const char* name = ""); -} +} diff --git a/library/cpp/mime/types/mime.cpp b/library/cpp/mime/types/mime.cpp index 706d776b24..b0e9209d31 100644 --- a/library/cpp/mime/types/mime.cpp +++ b/library/cpp/mime/types/mime.cpp @@ -1,40 +1,40 @@ #include "mime.h" - + #include <util/system/defaults.h> #include <util/generic/hash.h> #include <util/generic/strbuf.h> #include <util/generic/singleton.h> #include <util/generic/yexception.h> -#include <cctype> - +#include <cctype> + /* * MIME types */ class TMimeTypes { - // Constructor + // Constructor public: TMimeTypes(); - // Methods + // Methods public: const char* StrByExt(const char* ext) const; - MimeTypes MimeByStr(const char* str) const; + MimeTypes MimeByStr(const char* str) const; MimeTypes MimeByStr(const TStringBuf& str) const; const char* StrByMime(MimeTypes mime) const; - // Constants + // Constants public: static const size_t MAX_EXT_LEN = 11; // max length of supported extensions - // Helper methods + // Helper methods private: void SetContentTypes(); void SetExt(); - // Types + // Types private: struct TRecord { MimeTypes Mime; @@ -44,7 +44,7 @@ private: typedef THashMap<const char*, int> TRecordHash; - // Fields + // Fields private: static const TRecord Records[]; TRecordHash ContentTypes; @@ -53,38 +53,38 @@ private: const TMimeTypes::TRecord TMimeTypes::Records[] = { {MIME_UNKNOWN, nullptr, nullptr}, - {MIME_TEXT, "text/plain\0", "asc\0txt\0"}, - {MIME_HTML, "text/html\0", "html\0htm\0shtml\0"}, - {MIME_PDF, "application/pdf\0", "pdf\0"}, - {MIME_RTF, "text/rtf\0application/rtf\0", "rtf\0"}, - {MIME_DOC, "application/msword\0", "doc\0"}, - {MIME_MPEG, "audio/mpeg\0", "mp3\0mpa\0m2a\0mp2\0mpg\0mpga\0"}, + {MIME_TEXT, "text/plain\0", "asc\0txt\0"}, + {MIME_HTML, "text/html\0", "html\0htm\0shtml\0"}, + {MIME_PDF, "application/pdf\0", "pdf\0"}, + {MIME_RTF, "text/rtf\0application/rtf\0", "rtf\0"}, + {MIME_DOC, "application/msword\0", "doc\0"}, + {MIME_MPEG, "audio/mpeg\0", "mp3\0mpa\0m2a\0mp2\0mpg\0mpga\0"}, {MIME_XML, "text/xml\0application/xml\0", "xml\0"}, - {MIME_WML, "text/vnd.wap.wml\0", "wml\0"}, - {MIME_SWF, "application/x-shockwave-flash\0", "swf\0"}, - {MIME_XLS, "application/vnd.ms-excel\0", "xls\0"}, - {MIME_PPT, "application/vnd.ms-powerpoint\0", "ppt\0"}, - {MIME_IMAGE_JPG, "image/jpeg\0image/jpg\0", "jpeg\0jpg\0"}, - {MIME_IMAGE_PJPG, "image/pjpeg\0", "pjpeg\0"}, - {MIME_IMAGE_PNG, "image/png\0", "png\0"}, - {MIME_IMAGE_GIF, "image/gif\0", "gif\0"}, - {MIME_DOCX, "application/vnd.openxmlformats-officedocument.wordprocessingml.document\0", "docx\0"}, - {MIME_ODT, "application/vnd.oasis.opendocument.text\0", "odt\0"}, - {MIME_ODP, "application/vnd.oasis.opendocument.presentation\0", "odp\0"}, - {MIME_ODS, "application/vnd.oasis.opendocument.spreadsheet\0", "ods\0"}, + {MIME_WML, "text/vnd.wap.wml\0", "wml\0"}, + {MIME_SWF, "application/x-shockwave-flash\0", "swf\0"}, + {MIME_XLS, "application/vnd.ms-excel\0", "xls\0"}, + {MIME_PPT, "application/vnd.ms-powerpoint\0", "ppt\0"}, + {MIME_IMAGE_JPG, "image/jpeg\0image/jpg\0", "jpeg\0jpg\0"}, + {MIME_IMAGE_PJPG, "image/pjpeg\0", "pjpeg\0"}, + {MIME_IMAGE_PNG, "image/png\0", "png\0"}, + {MIME_IMAGE_GIF, "image/gif\0", "gif\0"}, + {MIME_DOCX, "application/vnd.openxmlformats-officedocument.wordprocessingml.document\0", "docx\0"}, + {MIME_ODT, "application/vnd.oasis.opendocument.text\0", "odt\0"}, + {MIME_ODP, "application/vnd.oasis.opendocument.presentation\0", "odp\0"}, + {MIME_ODS, "application/vnd.oasis.opendocument.spreadsheet\0", "ods\0"}, {MIME_UNKNOWN, nullptr, nullptr}, {MIME_IMAGE_BMP, "image/bmp\0image/x-ms-bmp\0image/x-windows-bmp\0", "bmp\0"}, - {MIME_WAV, "audio/x-wav\0", "wav\0"}, + {MIME_WAV, "audio/x-wav\0", "wav\0"}, {MIME_ARCHIVE, "application/x-archive\0", nullptr}, - {MIME_EXE, "application/exe\0application/octet-stream\0application/x-dosexec\0application/x-msdownload\0", "exe\0"}, - {MIME_ODG, "application/vnd.oasis.opendocument.graphics\0", "odg\0"}, - {MIME_GZIP, "application/x-gzip\0", "gz\0gzip\0"}, - {MIME_XLSX, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\0", "xlsx\0"}, - {MIME_PPTX, "application/vnd.openxmlformats-officedocument.presentationml.presentation\0", "pptx\0"}, - {MIME_JAVASCRIPT, "application/javascript\0text/javascript\0", "js\0"}, - {MIME_EPUB, "application/epub+zip\0", "epub\0"}, + {MIME_EXE, "application/exe\0application/octet-stream\0application/x-dosexec\0application/x-msdownload\0", "exe\0"}, + {MIME_ODG, "application/vnd.oasis.opendocument.graphics\0", "odg\0"}, + {MIME_GZIP, "application/x-gzip\0", "gz\0gzip\0"}, + {MIME_XLSX, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\0", "xlsx\0"}, + {MIME_PPTX, "application/vnd.openxmlformats-officedocument.presentationml.presentation\0", "pptx\0"}, + {MIME_JAVASCRIPT, "application/javascript\0text/javascript\0", "js\0"}, + {MIME_EPUB, "application/epub+zip\0", "epub\0"}, {MIME_TEX, "application/x-tex\0application/x-latex\0text/x-tex\0", "tex\0"}, - {MIME_JSON, "application/json\0", "json\0"}, + {MIME_JSON, "application/json\0", "json\0"}, {MIME_APK, "application/vnd.android.package-archive\0", "apk\0"}, {MIME_CSS, "text/css\0", "css\0"}, {MIME_IMAGE_WEBP, "image/webp\0", "webp\0"}, @@ -102,12 +102,12 @@ const TMimeTypes::TRecord TMimeTypes::Records[] = { {MIME_MAX, nullptr, nullptr}, // Additional records - {MIME_HTML, "application/xhtml+xml\0", "xhtml\0"}, + {MIME_HTML, "application/xhtml+xml\0", "xhtml\0"}, }; TMimeTypes::TMimeTypes() - : ContentTypes() - , Ext() + : ContentTypes() + , Ext() { SetContentTypes(); SetExt(); @@ -164,7 +164,7 @@ const char* TMimeTypes::StrByMime(MimeTypes mime) const { return Records[mime].ContentType; } -const char* mimetypeByExt(const char* fname, const char* check_ext) { +const char* mimetypeByExt(const char* fname, const char* check_ext) { const char* ext_p; if (fname == nullptr || *fname == 0 || (ext_p = strrchr(fname, '.')) == nullptr || strlen(ext_p) - 1 > TMimeTypes::MAX_EXT_LEN) { @@ -174,7 +174,7 @@ const char* mimetypeByExt(const char* fname, const char* check_ext) { char ext[TMimeTypes::MAX_EXT_LEN + 1]; size_t i; ext_p++; - for (i = 0; i < TMimeTypes::MAX_EXT_LEN && ext_p[i]; i++) + for (i = 0; i < TMimeTypes::MAX_EXT_LEN && ext_p[i]; i++) ext[i] = (char)tolower(ext_p[i]); ext[i] = 0; @@ -202,39 +202,39 @@ const char* strByMime(MimeTypes mime) { return Singleton<TMimeTypes>()->StrByMime(mime); } -const char* MimeNames[MIME_MAX] = { - "unknown", // MIME_UNKNOWN // 0 - "text", // MIME_TEXT // 1 - "html", // MIME_HTML // 2 - "pdf", // MIME_PDF // 3 - "rtf", // MIME_RTF // 4 - "doc", // MIME_DOC // 5 - "mpeg", // MIME_MPEG // 6 - "xml", // MIME_XML // 7 - "wap", // MIME_WML // 8 - "swf", // MIME_SWF // 9 - "xls", // MIME_XLS // 10 - "ppt", // MIME_PPT // 11 +const char* MimeNames[MIME_MAX] = { + "unknown", // MIME_UNKNOWN // 0 + "text", // MIME_TEXT // 1 + "html", // MIME_HTML // 2 + "pdf", // MIME_PDF // 3 + "rtf", // MIME_RTF // 4 + "doc", // MIME_DOC // 5 + "mpeg", // MIME_MPEG // 6 + "xml", // MIME_XML // 7 + "wap", // MIME_WML // 8 + "swf", // MIME_SWF // 9 + "xls", // MIME_XLS // 10 + "ppt", // MIME_PPT // 11 "jpg", // MIME_IMAGE_JPG // 12 "pjpg", // MIME_IMAGE_PJPG // 13 - "png", // MIME_IMAGE_PNG // 14 - "gif", // MIME_IMAGE_GIF // 15 - "docx", // MIME_DOCX // 16 - "odt", // MIME_ODT // 17 - "odp", // MIME_ODP // 18 - "ods", // MIME_ODS // 19 - "xmlhtml", // MIME_XHTMLXML // 20 - "bmp", // MIME_IMAGE_BMP // 21 - "wav", // MIME_WAV // 22 - "archive", // MIME_ARCHIVE // 23 - "exe", // MIME_EXE // 24 - "odg", // MIME_ODG // 25 - "gzip", // MIME_GZIP // 26 - "xlsx", // MIME_XLSX // 27 - "pptx", // MIME_PPTX // 28 - "js", // MIME_JAVASCRIPT // 29 - "epub", // MIME_EPUB // 30 - "tex", // MIME_TEX // 31 + "png", // MIME_IMAGE_PNG // 14 + "gif", // MIME_IMAGE_GIF // 15 + "docx", // MIME_DOCX // 16 + "odt", // MIME_ODT // 17 + "odp", // MIME_ODP // 18 + "ods", // MIME_ODS // 19 + "xmlhtml", // MIME_XHTMLXML // 20 + "bmp", // MIME_IMAGE_BMP // 21 + "wav", // MIME_WAV // 22 + "archive", // MIME_ARCHIVE // 23 + "exe", // MIME_EXE // 24 + "odg", // MIME_ODG // 25 + "gzip", // MIME_GZIP // 26 + "xlsx", // MIME_XLSX // 27 + "pptx", // MIME_PPTX // 28 + "js", // MIME_JAVASCRIPT // 29 + "epub", // MIME_EPUB // 30 + "tex", // MIME_TEX // 31 "json", // MIME_JSON // 32 "apk", // MIME_APK // 33 "css", // MIME_CSS // 34 diff --git a/library/cpp/mime/types/mime.h b/library/cpp/mime/types/mime.h index 05da389ea9..d28f3af588 100644 --- a/library/cpp/mime/types/mime.h +++ b/library/cpp/mime/types/mime.h @@ -1,49 +1,49 @@ #pragma once -#include <util/system/defaults.h> -#include <util/generic/strbuf.h> - +#include <util/system/defaults.h> +#include <util/generic/strbuf.h> + #include <cstring> - + enum MimeTypes { - MIME_UNKNOWN = 0, - MIME_TEXT = 1, - MIME_HTML = 2, - MIME_XHTMLXML = MIME_HTML, - MIME_PDF = 3, - MIME_RTF = 4, - MIME_DOC = 5, - MIME_MSWORD = MIME_DOC, - MIME_MPEG = 6, - MIME_XML = 7, - MIME_RSS = MIME_XML, - MIME_WML = 8, - MIME_SWF = 9, - MIME_FLASH = MIME_SWF, - MIME_XLS = 10, - MIME_EXCEL = MIME_XLS, - MIME_PPT = 11, - MIME_IMAGE_JPG = 12, + MIME_UNKNOWN = 0, + MIME_TEXT = 1, + MIME_HTML = 2, + MIME_XHTMLXML = MIME_HTML, + MIME_PDF = 3, + MIME_RTF = 4, + MIME_DOC = 5, + MIME_MSWORD = MIME_DOC, + MIME_MPEG = 6, + MIME_XML = 7, + MIME_RSS = MIME_XML, + MIME_WML = 8, + MIME_SWF = 9, + MIME_FLASH = MIME_SWF, + MIME_XLS = 10, + MIME_EXCEL = MIME_XLS, + MIME_PPT = 11, + MIME_IMAGE_JPG = 12, MIME_IMAGE_PJPG = 13, - MIME_IMAGE_PNG = 14, - MIME_IMAGE_GIF = 15, - MIME_DOCX = 16, - MIME_ODT = 17, - MIME_ODP = 18, - MIME_ODS = 19, + MIME_IMAGE_PNG = 14, + MIME_IMAGE_GIF = 15, + MIME_DOCX = 16, + MIME_ODT = 17, + MIME_ODP = 18, + MIME_ODS = 19, //MIME_XHTMLXML = 20, - MIME_IMAGE_BMP = 21, - MIME_WAV = 22, - MIME_ARCHIVE = 23, - MIME_EXE = 24, - MIME_ODG = 25, - MIME_GZIP = 26, - MIME_XLSX = 27, - MIME_PPTX = 28, + MIME_IMAGE_BMP = 21, + MIME_WAV = 22, + MIME_ARCHIVE = 23, + MIME_EXE = 24, + MIME_ODG = 25, + MIME_GZIP = 26, + MIME_XLSX = 27, + MIME_PPTX = 28, MIME_JAVASCRIPT = 29, - MIME_EPUB = 30, - MIME_TEX = 31, - MIME_JSON = 32, + MIME_EPUB = 30, + MIME_TEX = 31, + MIME_JSON = 32, MIME_APK = 33, MIME_CSS = 34, MIME_IMAGE_WEBP = 35, @@ -61,10 +61,10 @@ enum MimeTypes { MIME_MAX }; -extern const char* MimeNames[MIME_MAX]; +extern const char* MimeNames[MIME_MAX]; const char* mimetypeByExt(const char* fname, const char* check_ext = nullptr); -MimeTypes mimeByStr(const char* mimeStr); +MimeTypes mimeByStr(const char* mimeStr); MimeTypes mimeByStr(const TStringBuf& mimeStr); const char* strByMime(MimeTypes mime); diff --git a/library/cpp/monlib/counters/timer.h b/library/cpp/monlib/counters/timer.h index 03dfb35337..29bdc17a98 100644 --- a/library/cpp/monlib/counters/timer.h +++ b/library/cpp/monlib/counters/timer.h @@ -2,8 +2,8 @@ #include "histogram.h" -#include <util/generic/scope.h> - +#include <util/generic/scope.h> + #include <chrono> namespace NMonitoring { @@ -113,8 +113,8 @@ namespace NMonitoring { using TClock = std::chrono::high_resolution_clock; auto start = TClock::now(); - - Y_SCOPE_EXIT(this, start) { + + Y_SCOPE_EXIT(this, start) { RecordValue(TClock::now() - start); }; diff --git a/library/cpp/monlib/dynamic_counters/counters.h b/library/cpp/monlib/dynamic_counters/counters.h index dc178cfbe0..68787ab8df 100644 --- a/library/cpp/monlib/dynamic_counters/counters.h +++ b/library/cpp/monlib/dynamic_counters/counters.h @@ -20,7 +20,7 @@ namespace NMonitoring { struct ICountableConsumer; - struct TCountableBase: public TAtomicRefCount<TCountableBase> { + struct TCountableBase: public TAtomicRefCount<TCountableBase> { // Private means that the object must not be serialized unless the consumer // has explicitly specified this by setting its Visibility to Private. // diff --git a/library/cpp/monlib/encode/spack/compression.cpp b/library/cpp/monlib/encode/spack/compression.cpp index 0d2152fc85..c5c9045521 100644 --- a/library/cpp/monlib/encode/spack/compression.cpp +++ b/library/cpp/monlib/encode/spack/compression.cpp @@ -3,7 +3,7 @@ #include <util/generic/buffer.h> #include <util/generic/cast.h> #include <util/generic/ptr.h> -#include <util/generic/scope.h> +#include <util/generic/scope.h> #include <util/generic/size_literals.h> #include <util/stream/format.h> #include <util/stream/output.h> @@ -285,15 +285,15 @@ namespace NMonitoring { void FinishAndWriteEmptyFrame() override { if (Out_) { - Y_DEFER { + Y_DEFER { Out_ = nullptr; - }; - - if (!Uncompressed_.Empty()) { - WriteCompressedFrame(); + }; + + if (!Uncompressed_.Empty()) { + WriteCompressedFrame(); } - - WriteEmptyFrame(); + + WriteEmptyFrame(); } } diff --git a/library/cpp/monlib/service/service.cpp b/library/cpp/monlib/service/service.cpp index 929efbf816..98c314d8d6 100644 --- a/library/cpp/monlib/service/service.cpp +++ b/library/cpp/monlib/service/service.cpp @@ -94,7 +94,7 @@ namespace NMonitoring { THttpRequestHeader Header; const THttpHeaders* Headers = nullptr; THttpURL Url; - TCgiParameters CgiParams; + TCgiParameters CgiParams; TCgiParameters PostParams; TBufferOutput PostContent; const NAddr::IRemoteAddr* RemoteAddr = nullptr; @@ -223,8 +223,8 @@ namespace NMonitoring { if (!Start()) { const auto& opts = THttpServer::Options(); TNetworkAddress addr = opts.Host - ? TNetworkAddress(opts.Host, opts.Port) - : TNetworkAddress(opts.Port); + ? TNetworkAddress(opts.Host, opts.Port) + : TNetworkAddress(opts.Port); ythrow TSystemError(GetErrorCode()) << addr; } } diff --git a/library/cpp/monlib/service/service.h b/library/cpp/monlib/service/service.h index 2f66dddaf8..7a04808092 100644 --- a/library/cpp/monlib/service/service.h +++ b/library/cpp/monlib/service/service.h @@ -19,7 +19,7 @@ namespace NMonitoring { } virtual const char* GetURI() const = 0; virtual const char* GetPath() const = 0; - virtual const TCgiParameters& GetParams() const = 0; + virtual const TCgiParameters& GetParams() const = 0; virtual const TCgiParameters& GetPostParams() const = 0; virtual TStringBuf GetPostContent() const = 0; virtual HTTP_METHOD GetMethod() const = 0; diff --git a/library/cpp/object_factory/object_factory.h b/library/cpp/object_factory/object_factory.h index 96cc11bcfd..9a91fbee8e 100644 --- a/library/cpp/object_factory/object_factory.h +++ b/library/cpp/object_factory/object_factory.h @@ -4,121 +4,121 @@ #include <util/system/rwlock.h> #include <util/generic/map.h> #include <util/generic/set.h> -#include <util/generic/singleton.h> +#include <util/generic/singleton.h> #include <util/generic/yexception.h> namespace NObjectFactory { - template <class TProduct, class... TArgs> - class IFactoryObjectCreator { - public: + template <class TProduct, class... TArgs> + class IFactoryObjectCreator { + public: virtual TProduct* Create(TArgs... args) const = 0; - virtual ~IFactoryObjectCreator() { - } - }; - - template <class TProduct> - class IFactoryObjectCreator<TProduct, void> { - public: - virtual TProduct* Create(void) const = 0; - virtual ~IFactoryObjectCreator() { - } - }; - -#define FACTORY_OBJECT_NAME(Name) \ - static TString GetTypeName() { \ - return #Name; \ - } \ - virtual TString GetType() const override { \ - return #Name; \ - } - - template <class TBaseProduct, class TDerivedProduct, class... TArgs> - class TFactoryObjectCreator: public IFactoryObjectCreator<TBaseProduct, TArgs...> { + virtual ~IFactoryObjectCreator() { + } + }; + + template <class TProduct> + class IFactoryObjectCreator<TProduct, void> { + public: + virtual TProduct* Create(void) const = 0; + virtual ~IFactoryObjectCreator() { + } + }; + +#define FACTORY_OBJECT_NAME(Name) \ + static TString GetTypeName() { \ + return #Name; \ + } \ + virtual TString GetType() const override { \ + return #Name; \ + } + + template <class TBaseProduct, class TDerivedProduct, class... TArgs> + class TFactoryObjectCreator: public IFactoryObjectCreator<TBaseProduct, TArgs...> { TDerivedProduct* Create(TArgs... args) const override { return new TDerivedProduct(std::forward<TArgs>(args)...); - } - }; - - template <class TBaseProduct, class TDerivedProduct> - class TFactoryObjectCreator<TBaseProduct, TDerivedProduct, void>: public IFactoryObjectCreator<TBaseProduct, void> { - TDerivedProduct* Create() const override { - return new TDerivedProduct(); - } - }; - - template <class P, class K, class... TArgs> - class IObjectFactory { - public: - typedef P TProduct; - typedef K TKey; - - public: - template <class TDerivedProduct> + } + }; + + template <class TBaseProduct, class TDerivedProduct> + class TFactoryObjectCreator<TBaseProduct, TDerivedProduct, void>: public IFactoryObjectCreator<TBaseProduct, void> { + TDerivedProduct* Create() const override { + return new TDerivedProduct(); + } + }; + + template <class P, class K, class... TArgs> + class IObjectFactory { + public: + typedef P TProduct; + typedef K TKey; + + public: + template <class TDerivedProduct> void Register(const TKey& key, IFactoryObjectCreator<TProduct, TArgs...>* creator) { - if (!creator) + if (!creator) ythrow yexception() << "Please specify non-null creator for " << key; - TWriteGuard guard(CreatorsLock); - if (!Creators.insert(typename ICreators::value_type(key, creator)).second) - ythrow yexception() << "Product with key " << key << " already registered"; - } + TWriteGuard guard(CreatorsLock); + if (!Creators.insert(typename ICreators::value_type(key, creator)).second) + ythrow yexception() << "Product with key " << key << " already registered"; + } template <class TDerivedProduct> void Register(const TKey& key) { Register<TDerivedProduct>(key, new TFactoryObjectCreator<TProduct, TDerivedProduct, TArgs...>); } - void GetKeys(TSet<TKey>& keys) const { - TReadGuard guard(CreatorsLock); - keys.clear(); - for (typename ICreators::const_iterator i = Creators.begin(), e = Creators.end(); i != e; ++i) { - keys.insert(i->first); - } - } - - protected: - IFactoryObjectCreator<TProduct, TArgs...>* GetCreator(const TKey& key) const { - TReadGuard guard(CreatorsLock); - typename ICreators::const_iterator i = Creators.find(key); - return i == Creators.end() ? nullptr : i->second.Get(); - } - - bool HasImpl(const TKey& key) const { - TReadGuard guard(CreatorsLock); - return Creators.find(key) != Creators.end(); - } - - private: - typedef TSimpleSharedPtr<IFactoryObjectCreator<TProduct, TArgs...>> ICreatorPtr; - typedef TMap<TKey, ICreatorPtr> ICreators; - ICreators Creators; - TRWMutex CreatorsLock; - }; - - template <class TProduct, class TKey> - class TObjectFactory: public IObjectFactory<TProduct, TKey, void> { - public: - TProduct* Create(const TKey& key) const { - IFactoryObjectCreator<TProduct, void>* creator = IObjectFactory<TProduct, TKey, void>::GetCreator(key); - return creator == nullptr ? nullptr : creator->Create(); - } - - static TString KeysDebugString() { - TSet<TString> keys; - Singleton<TObjectFactory<TProduct, TKey>>()->GetKeys(keys); - TString keysStr; - for (auto&& k : keys) { - keysStr += k + " "; - } - return keysStr; - } + void GetKeys(TSet<TKey>& keys) const { + TReadGuard guard(CreatorsLock); + keys.clear(); + for (typename ICreators::const_iterator i = Creators.begin(), e = Creators.end(); i != e; ++i) { + keys.insert(i->first); + } + } + + protected: + IFactoryObjectCreator<TProduct, TArgs...>* GetCreator(const TKey& key) const { + TReadGuard guard(CreatorsLock); + typename ICreators::const_iterator i = Creators.find(key); + return i == Creators.end() ? nullptr : i->second.Get(); + } + + bool HasImpl(const TKey& key) const { + TReadGuard guard(CreatorsLock); + return Creators.find(key) != Creators.end(); + } + + private: + typedef TSimpleSharedPtr<IFactoryObjectCreator<TProduct, TArgs...>> ICreatorPtr; + typedef TMap<TKey, ICreatorPtr> ICreators; + ICreators Creators; + TRWMutex CreatorsLock; + }; + + template <class TProduct, class TKey> + class TObjectFactory: public IObjectFactory<TProduct, TKey, void> { + public: + TProduct* Create(const TKey& key) const { + IFactoryObjectCreator<TProduct, void>* creator = IObjectFactory<TProduct, TKey, void>::GetCreator(key); + return creator == nullptr ? nullptr : creator->Create(); + } + + static TString KeysDebugString() { + TSet<TString> keys; + Singleton<TObjectFactory<TProduct, TKey>>()->GetKeys(keys); + TString keysStr; + for (auto&& k : keys) { + keysStr += k + " "; + } + return keysStr; + } static TProduct* Construct(const TKey& key, const TKey& defKey) { - TProduct* result = Singleton<TObjectFactory<TProduct, TKey>>()->Create(key); - if (!result && !!defKey) { - result = Singleton<TObjectFactory<TProduct, TKey>>()->Create(defKey); - } - return result; + TProduct* result = Singleton<TObjectFactory<TProduct, TKey>>()->Create(key); + if (!result && !!defKey) { + result = Singleton<TObjectFactory<TProduct, TKey>>()->Create(defKey); + } + return result; } static TProduct* Construct(const TKey& key) { @@ -137,13 +137,13 @@ namespace NObjectFactory { return THolder<TProduct>(Construct(std::forward<Args>(args)...)); } - static bool Has(const TKey& key) { - return Singleton<TObjectFactory<TProduct, TKey>>()->HasImpl(key); + static bool Has(const TKey& key) { + return Singleton<TObjectFactory<TProduct, TKey>>()->HasImpl(key); } static void GetRegisteredKeys(TSet<TKey>& keys) { - return Singleton<TObjectFactory<TProduct, TKey>>()->GetKeys(keys); - } + return Singleton<TObjectFactory<TProduct, TKey>>()->GetKeys(keys); + } static TSet<TKey> GetRegisteredKeys() { TSet<TKey> keys; @@ -162,39 +162,39 @@ namespace NObjectFactory { return fileredKeys; } - template <class Product> - class TRegistrator { - public: + template <class Product> + class TRegistrator { + public: TRegistrator(const TKey& key, IFactoryObjectCreator<TProduct, void>* creator) { - Singleton<TObjectFactory<TProduct, TKey>>()->template Register<Product>(key, creator); - } + Singleton<TObjectFactory<TProduct, TKey>>()->template Register<Product>(key, creator); + } TRegistrator(const TKey& key) { Singleton<TObjectFactory<TProduct, TKey>>()->template Register<Product>(key); } - TRegistrator() - : TRegistrator(Product::GetTypeName()) - { - } - }; - }; + TRegistrator() + : TRegistrator(Product::GetTypeName()) + { + } + }; + }; - template <class TProduct, class TKey, class... TArgs> - class TParametrizedObjectFactory: public IObjectFactory<TProduct, TKey, TArgs...> { + template <class TProduct, class TKey, class... TArgs> + class TParametrizedObjectFactory: public IObjectFactory<TProduct, TKey, TArgs...> { public: - TProduct* Create(const TKey& key, TArgs... args) const { - IFactoryObjectCreator<TProduct, TArgs...>* creator = IObjectFactory<TProduct, TKey, TArgs...>::GetCreator(key); + TProduct* Create(const TKey& key, TArgs... args) const { + IFactoryObjectCreator<TProduct, TArgs...>* creator = IObjectFactory<TProduct, TKey, TArgs...>::GetCreator(key); return creator == nullptr ? nullptr : creator->Create(std::forward<TArgs>(args)...); } - static bool Has(const TKey& key) { - return Singleton<TParametrizedObjectFactory<TProduct, TKey, TArgs...>>()->HasImpl(key); + static bool Has(const TKey& key) { + return Singleton<TParametrizedObjectFactory<TProduct, TKey, TArgs...>>()->HasImpl(key); } - static TProduct* Construct(const TKey& key, TArgs... args) { + static TProduct* Construct(const TKey& key, TArgs... args) { return Singleton<TParametrizedObjectFactory<TProduct, TKey, TArgs...>>()->Create(key, std::forward<TArgs>(args)...); - } + } template <class... Args> static THolder<TProduct> VerifiedConstruct(Args&&... args) { @@ -208,9 +208,9 @@ namespace NObjectFactory { return THolder<TProduct>(Construct(std::forward<Args>(args)...)); } - static void GetRegisteredKeys(TSet<TKey>& keys) { - return Singleton<TParametrizedObjectFactory<TProduct, TKey, TArgs...>>()->GetKeys(keys); - } + static void GetRegisteredKeys(TSet<TKey>& keys) { + return Singleton<TParametrizedObjectFactory<TProduct, TKey, TArgs...>>()->GetKeys(keys); + } static TSet<TKey> GetRegisteredKeys() { TSet<TKey> keys; @@ -218,26 +218,26 @@ namespace NObjectFactory { return keys; } - template <class Product> - class TRegistrator { - public: + template <class Product> + class TRegistrator { + public: TRegistrator(const TKey& key, IFactoryObjectCreator<TProduct, TArgs...>* creator) { - Singleton<TParametrizedObjectFactory<TProduct, TKey, TArgs...>>()->template Register<Product>(key, creator); - } + Singleton<TParametrizedObjectFactory<TProduct, TKey, TArgs...>>()->template Register<Product>(key, creator); + } TRegistrator(const TKey& key) { Singleton<TParametrizedObjectFactory<TProduct, TKey, TArgs...>>()->template Register<Product>(key); } - TRegistrator() - : TRegistrator(Product::GetTypeName()) - { - } + TRegistrator() + : TRegistrator(Product::GetTypeName()) + { + } TString GetName() const { return Product::GetTypeName(); } - }; + }; }; } diff --git a/library/cpp/object_factory/object_factory_ut.cpp b/library/cpp/object_factory/object_factory_ut.cpp index 06fb0739ff..16edb03e3b 100644 --- a/library/cpp/object_factory/object_factory_ut.cpp +++ b/library/cpp/object_factory/object_factory_ut.cpp @@ -14,44 +14,44 @@ struct TArgument { class ICommonInterface { public: - virtual ~ICommonInterface() { - } + virtual ~ICommonInterface() { + } virtual TString GetValue() const = 0; }; -class TDirectOrder: public ICommonInterface { +class TDirectOrder: public ICommonInterface { public: TDirectOrder(const TString& provider, float factor, TArgument& argument) : Provider(provider) , Factor(factor) , Argument(argument) - { - } + { + } TString GetValue() const override { return Provider + ToString(Factor) + Argument.Name; } - + private: const TString Provider; const float Factor; const TArgument Argument; }; -class TInverseOrder: public ICommonInterface { +class TInverseOrder: public ICommonInterface { public: TInverseOrder(const TString& provider, float factor, TArgument& argument) : Provider(provider) , Factor(factor) , Argument(argument) - { - } + { + } TString GetValue() const override { return Argument.Name + ToString(Factor) + Provider; } - + private: const TString Provider; const float Factor; @@ -144,8 +144,8 @@ static TTestFactory::TRegistrator<TDirectOrderDifferentSignature> DirectDs("dire Y_UNIT_TEST_SUITE(TestObjectFactory) { Y_UNIT_TEST(TestParametrized) { - TArgument directArg{"Name", nullptr}; - TArgument inverseArg{"Fake", nullptr}; + TArgument directArg{"Name", nullptr}; + TArgument inverseArg{"Fake", nullptr}; THolder<ICommonInterface> direct(TTestFactory::Construct("direct", "prov", 0.42, directArg)); THolder<ICommonInterface> inverse(TTestFactory::Construct("inverse", "prov2", 1, inverseArg)); diff --git a/library/cpp/on_disk/chunks/chunked_helpers.cpp b/library/cpp/on_disk/chunks/chunked_helpers.cpp index b7adba2753..9671529fe8 100644 --- a/library/cpp/on_disk/chunks/chunked_helpers.cpp +++ b/library/cpp/on_disk/chunks/chunked_helpers.cpp @@ -12,7 +12,7 @@ TBlob GetBlock(const TBlob& blob, size_t index) { /*************************** TNamedChunkedDataReader ***************************/ -static const char* NamedChunkedDataMagic = "NamedChunkedData"; +static const char* NamedChunkedDataMagic = "NamedChunkedData"; TNamedChunkedDataReader::TNamedChunkedDataReader(const TBlob& blob) : TChunkedDataReader(blob) @@ -25,7 +25,7 @@ TNamedChunkedDataReader::TNamedChunkedDataReader(const TBlob& blob) if (GetBlockLen(block) < magicLen || memcmp(GetBlock(block), NamedChunkedDataMagic, magicLen) != 0) throw yexception() << "Not a valid named chunked data file"; - TMemoryInput input(static_cast<const char*>(GetBlock(block)) + magicLen, GetBlockLen(block) - magicLen); + TMemoryInput input(static_cast<const char*>(GetBlock(block)) + magicLen, GetBlockLen(block) - magicLen); Load(&input, Names); size_t index = 0; diff --git a/library/cpp/on_disk/chunks/chunked_helpers.h b/library/cpp/on_disk/chunks/chunked_helpers.h index 5fa96afdca..4f5421c71d 100644 --- a/library/cpp/on_disk/chunks/chunked_helpers.h +++ b/library/cpp/on_disk/chunks/chunked_helpers.h @@ -2,9 +2,9 @@ #include <util/generic/vector.h> #include <util/generic/buffer.h> -#include <util/generic/hash_set.h> +#include <util/generic/hash_set.h> #include <util/generic/cast.h> -#include <util/generic/ymath.h> +#include <util/generic/ymath.h> #include <util/memory/blob.h> #include <util/stream/buffer.h> #include <util/stream/mem.h> @@ -17,7 +17,7 @@ #include <cmath> #include <cstddef> -template <typename T> +template <typename T> class TYVector { private: ui32 Size; @@ -26,7 +26,7 @@ private: public: TYVector(const TBlob& blob) : Size(IntegerCast<ui32>(ReadUnaligned<ui64>(blob.Data()))) - , Data((const T*)((const char*)blob.Data() + sizeof(ui64))) + , Data((const T*)((const char*)blob.Data() + sizeof(ui64))) { } @@ -45,13 +45,13 @@ public: } size_t RealSize() const { - return sizeof(ui64) + Size * sizeof(T); + return sizeof(ui64) + Size * sizeof(T); } ~TYVector() = default; }; -template <typename T> +template <typename T> class TYVectorWriter { private: TVector<T> Vector; @@ -90,21 +90,21 @@ public: void Resize(size_t size) { Vector.resize(size); } - + void Resize(size_t size, const T& value) { Vector.resize(size, value); } }; -template <typename T, bool> +template <typename T, bool> struct TYVectorG; -template <typename X> +template <typename X> struct TYVectorG<X, false> { typedef TYVector<X> T; }; -template <typename X> +template <typename X> struct TYVectorG<X, true> { typedef TYVectorWriter<X> T; }; @@ -116,18 +116,18 @@ struct TIsMemsetThisWithZeroesSupported { }; }; -#define MEMSET_THIS_WITH_ZEROES_SUPPORTED(type) \ - template <> \ - struct TIsMemsetThisWithZeroesSupported<type> { \ - enum { \ - Result = true \ - }; \ +#define MEMSET_THIS_WITH_ZEROES_SUPPORTED(type) \ + template <> \ + struct TIsMemsetThisWithZeroesSupported<type> { \ + enum { \ + Result = true \ + }; \ }; class TPlainHashCommon { protected: #pragma pack(push, 8) - template <typename TKey, typename TValue> + template <typename TKey, typename TValue> class TPackedPair { private: typedef TPackedPair<TKey, TValue> TThis; @@ -135,8 +135,8 @@ protected: TValue Value; private: - static_assert(TIsMemsetThisWithZeroesSupported<TKey>::Result, "expect TIsMemsetThisWithZeroesSupported<TKey>::Result"); - static_assert(TIsMemsetThisWithZeroesSupported<TValue>::Result, "expect TIsMemsetThisWithZeroesSupported<TValue>::Result"); + static_assert(TIsMemsetThisWithZeroesSupported<TKey>::Result, "expect TIsMemsetThisWithZeroesSupported<TKey>::Result"); + static_assert(TIsMemsetThisWithZeroesSupported<TValue>::Result, "expect TIsMemsetThisWithZeroesSupported<TValue>::Result"); /// to aviod uninitialized bytes void Init(const TKey& key, const TValue& value) { @@ -217,9 +217,9 @@ protected: } }; #pragma pack(pop) - static_assert(8 == sizeof(TInterval), "expect 8 == sizeof(TInterval)"); + static_assert(8 == sizeof(TInterval), "expect 8 == sizeof(TInterval)"); - template <typename TKey> + template <typename TKey> static ui32 KeyHash(typename TTypeTraits<TKey>::TFuncParam key, ui16 bits) { Y_ASSERT(bits < 32); const ui32 res = ui32(key) & ((ui32(1) << bits) - 1); @@ -229,7 +229,7 @@ protected: } }; -template <typename TKey, typename TValue> +template <typename TKey, typename TValue> class TPlainHashWriter : TPlainHashCommon { private: typedef TPackedPair<TKey, TValue> TKeyValuePair; @@ -250,7 +250,7 @@ private: public: void Add(const TKey& key, const TValue& value) { - Data.push_back(TKeyValuePair(key, value)); + Data.push_back(TKeyValuePair(key, value)); } void Save(IOutputStream& out) const { @@ -262,8 +262,8 @@ public: ui16 bits; if (!Data.empty()) { - bits = (ui16)(log((float)Data.size()) / log(2.f)); - while ((bits < 22) && !IsPlainEnought(bits)) + bits = (ui16)(log((float)Data.size()) / log(2.f)); + while ((bits < 22) && !IsPlainEnought(bits)) ++bits; } else { bits = 0; @@ -298,7 +298,7 @@ public: } }; -template <typename TKey, typename TValue> +template <typename TKey, typename TValue> class TPlainHash : TPlainHashCommon { private: typedef TPackedPair<TKey, TValue> TKeyValuePair; @@ -321,9 +321,9 @@ private: return (const TKeyValuePair*)(GetIntervals() + (1ULL << GetBits())); } - template <typename T> + template <typename T> void Init(const T* p) { - static_assert(sizeof(T) == 1, "expect sizeof(T) == 1"); + static_assert(sizeof(T) == 1, "expect sizeof(T) == 1"); P = reinterpret_cast<const char*>(p); #ifndef NDEBUG ui16 version = ReadUnaligned<ui16>(p); @@ -383,24 +383,24 @@ public: } size_t ByteSize() const { - return 12 + sizeof(TInterval) * (size_t(1) << GetBits()) + sizeof(TKeyValuePair) * GetSize(); + return 12 + sizeof(TInterval) * (size_t(1) << GetBits()) + sizeof(TKeyValuePair) * GetSize(); } }; -template <typename Key, typename Value, bool> +template <typename Key, typename Value, bool> struct TPlainHashG; -template <typename Key, typename Value> +template <typename Key, typename Value> struct TPlainHashG<Key, Value, false> { typedef TPlainHash<Key, Value> T; }; -template <typename Key, typename Value> +template <typename Key, typename Value> struct TPlainHashG<Key, Value, true> { typedef TPlainHashWriter<Key, Value> T; }; -template <typename T> +template <typename T> class TSingleValue { private: const T* Value; @@ -417,7 +417,7 @@ public: } }; -template <typename T> +template <typename T> class TSingleValueWriter { private: T Value; @@ -425,7 +425,7 @@ private: public: TSingleValueWriter() = default; - TSingleValueWriter(const T& value) + TSingleValueWriter(const T& value) : Value(value) { } @@ -441,13 +441,13 @@ public: TBlob GetBlock(const TBlob& data, size_t index); -template <class T> +template <class T> void WriteBlock(TChunkedDataWriter& writer, const T& t) { writer.NewBlock(); t.Save(writer); } -template <class T> +template <class T> void WriteBlock(TChunkedDataWriter& writer, T& t) { writer.NewBlock(); t.Save(writer); @@ -455,78 +455,78 @@ void WriteBlock(TChunkedDataWriter& writer, T& t) { // Extends TChunkedDataWriter, allowing user to name blocks with arbitrary strings. class TNamedChunkedDataWriter: public TChunkedDataWriter { -public: - TNamedChunkedDataWriter(IOutputStream& slave); - ~TNamedChunkedDataWriter() override; +public: + TNamedChunkedDataWriter(IOutputStream& slave); + ~TNamedChunkedDataWriter() override; - // Start a new unnamed block, overrides TChunkedDataReader::NewBlock(). - void NewBlock(); + // Start a new unnamed block, overrides TChunkedDataReader::NewBlock(). + void NewBlock(); - // Start a new block with given name (possibly empty, in which case block is unnamed). - // Throws an exception if name is a duplicate. - void NewBlock(const TString& name); + // Start a new block with given name (possibly empty, in which case block is unnamed). + // Throws an exception if name is a duplicate. + void NewBlock(const TString& name); - void WriteFooter(); + void WriteFooter(); -private: - TVector<TString> Names; - THashMap<TString, size_t> NameToIndex; +private: + TVector<TString> Names; + THashMap<TString, size_t> NameToIndex; }; -class TNamedChunkedDataReader: public TChunkedDataReader { -public: - TNamedChunkedDataReader(const TBlob& blob); - - inline bool HasBlock(const char* name) const { - return NameToIndex.find(name) != NameToIndex.end(); - } - - inline size_t GetIndexByName(const char* name) const { - THashMap<TString, size_t>::const_iterator it = NameToIndex.find(name); - if (it == NameToIndex.end()) - throw yexception() << "Block \"" << name << "\" is not found"; - else - return it->second; - } - - // Returns number of blocks written to the file by user of TNamedChunkedDataReader. - inline size_t GetBlocksCount() const { - // Last block is for internal usage - return TChunkedDataReader::GetBlocksCount() - 1; - } - - inline const char* GetBlockName(size_t index) const { - Y_ASSERT(index < GetBlocksCount()); +class TNamedChunkedDataReader: public TChunkedDataReader { +public: + TNamedChunkedDataReader(const TBlob& blob); + + inline bool HasBlock(const char* name) const { + return NameToIndex.find(name) != NameToIndex.end(); + } + + inline size_t GetIndexByName(const char* name) const { + THashMap<TString, size_t>::const_iterator it = NameToIndex.find(name); + if (it == NameToIndex.end()) + throw yexception() << "Block \"" << name << "\" is not found"; + else + return it->second; + } + + // Returns number of blocks written to the file by user of TNamedChunkedDataReader. + inline size_t GetBlocksCount() const { + // Last block is for internal usage + return TChunkedDataReader::GetBlocksCount() - 1; + } + + inline const char* GetBlockName(size_t index) const { + Y_ASSERT(index < GetBlocksCount()); return Names[index].data(); - } - - inline const void* GetBlockByName(const char* name) const { - return GetBlock(GetIndexByName(name)); - } - - inline size_t GetBlockLenByName(const char* name) const { - return GetBlockLen(GetIndexByName(name)); - } - - inline TBlob GetBlobByName(const char* name) const { - size_t id = GetIndexByName(name); - return TBlob::NoCopy(GetBlock(id), GetBlockLen(id)); - } - - inline bool GetBlobByName(const char* name, TBlob& blob) const { - THashMap<TString, size_t>::const_iterator it = NameToIndex.find(name); - if (it == NameToIndex.end()) - return false; - blob = TBlob::NoCopy(GetBlock(it->second), GetBlockLen(it->second)); - return true; - } - -private: - TVector<TString> Names; - THashMap<TString, size_t> NameToIndex; + } + + inline const void* GetBlockByName(const char* name) const { + return GetBlock(GetIndexByName(name)); + } + + inline size_t GetBlockLenByName(const char* name) const { + return GetBlockLen(GetIndexByName(name)); + } + + inline TBlob GetBlobByName(const char* name) const { + size_t id = GetIndexByName(name); + return TBlob::NoCopy(GetBlock(id), GetBlockLen(id)); + } + + inline bool GetBlobByName(const char* name, TBlob& blob) const { + THashMap<TString, size_t>::const_iterator it = NameToIndex.find(name); + if (it == NameToIndex.end()) + return false; + blob = TBlob::NoCopy(GetBlock(it->second), GetBlockLen(it->second)); + return true; + } + +private: + TVector<TString> Names; + THashMap<TString, size_t> NameToIndex; }; -template <class T> +template <class T> struct TSaveLoadVectorNonPodElement { static inline void Save(IOutputStream* out, const T& t) { TSerializer<T>::Save(out, t); @@ -538,7 +538,7 @@ struct TSaveLoadVectorNonPodElement { } }; -template <class T, bool isPod> +template <class T, bool isPod> class TVectorTakingIntoAccountThePodType { private: ui64 SizeofOffsets; @@ -550,7 +550,7 @@ public: SizeofOffsets = ReadUnaligned<ui64>(blob.Begin()); Y_ASSERT(SizeofOffsets > 0); Offsets = reinterpret_cast<const ui64*>(blob.Begin() + sizeof(ui64)); - Data = reinterpret_cast<const char*>(blob.Begin() + sizeof(ui64) + SizeofOffsets * sizeof(ui64)); + Data = reinterpret_cast<const char*>(blob.Begin() + sizeof(ui64) + SizeofOffsets * sizeof(ui64)); } size_t GetSize() const { @@ -576,11 +576,11 @@ public: } size_t RealSize() const { - return sizeof(ui64) * (SizeofOffsets + 1) + ReadUnaligned<ui64>(Offsets + SizeofOffsets - 1); + return sizeof(ui64) * (SizeofOffsets + 1) + ReadUnaligned<ui64>(Offsets + SizeofOffsets - 1); } }; -template <class T, bool isPod> +template <class T, bool isPod> class TVectorTakingIntoAccountThePodTypeWriter : TNonCopyable { private: typedef TVector<ui64> TOffsets; @@ -613,8 +613,8 @@ public: } }; -template <class T> -class TVectorTakingIntoAccountThePodType<T, true>: public TYVector<T> { +template <class T> +class TVectorTakingIntoAccountThePodType<T, true>: public TYVector<T> { public: TVectorTakingIntoAccountThePodType(const TBlob& blob) : TYVector<T>(blob) @@ -622,14 +622,14 @@ public: } }; -template <class T> -class TVectorTakingIntoAccountThePodTypeWriter<T, true>: public TYVectorWriter<T> { +template <class T> +class TVectorTakingIntoAccountThePodTypeWriter<T, true>: public TYVectorWriter<T> { }; -template <typename T> -class TGeneralVector: public TVectorTakingIntoAccountThePodType<T, TTypeTraits<T>::IsPod> { +template <typename T> +class TGeneralVector: public TVectorTakingIntoAccountThePodType<T, TTypeTraits<T>::IsPod> { typedef TVectorTakingIntoAccountThePodType<T, TTypeTraits<T>::IsPod> TBase; - + public: TGeneralVector(const TBlob& blob) : TBase(blob) @@ -637,24 +637,24 @@ public: } }; -template <typename T> -class TGeneralVectorWriter: public TVectorTakingIntoAccountThePodTypeWriter<T, TTypeTraits<T>::IsPod> { +template <typename T> +class TGeneralVectorWriter: public TVectorTakingIntoAccountThePodTypeWriter<T, TTypeTraits<T>::IsPod> { }; -template <typename TItem, bool> +template <typename TItem, bool> struct TGeneralVectorG; -template <typename TItem> +template <typename TItem> struct TGeneralVectorG<TItem, false> { typedef TGeneralVector<TItem> T; }; -template <typename TItem> +template <typename TItem> struct TGeneralVectorG<TItem, true> { typedef TGeneralVectorWriter<TItem> T; }; -template <> +template <> struct TSaveLoadVectorNonPodElement<TString> { static inline void Save(IOutputStream* out, const TString& s) { out->Write(s.data(), s.size() + 1); @@ -666,8 +666,8 @@ struct TSaveLoadVectorNonPodElement<TString> { } }; -template <bool G> -struct TStringsVectorG: public TGeneralVectorG<TString, G> { +template <bool G> +struct TStringsVectorG: public TGeneralVectorG<TString, G> { }; using TStringsVector = TGeneralVector<TString>; diff --git a/library/cpp/on_disk/chunks/chunks_ut.cpp b/library/cpp/on_disk/chunks/chunks_ut.cpp index f727647f7f..4161851f7d 100644 --- a/library/cpp/on_disk/chunks/chunks_ut.cpp +++ b/library/cpp/on_disk/chunks/chunks_ut.cpp @@ -17,7 +17,7 @@ struct TPodStruct { } }; /// And its serialization -template <> +template <> struct TSaveLoadVectorNonPodElement<TPodStruct> { typedef TPodStruct TItem; static inline void Save(IOutputStream* out, const TItem& item) { @@ -32,14 +32,14 @@ struct TSaveLoadVectorNonPodElement<TPodStruct> { } }; -class TChunkedHelpersTest: public TTestBase { +class TChunkedHelpersTest: public TTestBase { UNIT_TEST_SUITE(TChunkedHelpersTest); - UNIT_TEST(TestHash) - UNIT_TEST(TestGeneralVector) - UNIT_TEST(TestStrings); - UNIT_TEST(TestNamedChunkedData); + UNIT_TEST(TestHash) + UNIT_TEST(TestGeneralVector) + UNIT_TEST(TestStrings); + UNIT_TEST(TestNamedChunkedData); UNIT_TEST_SUITE_END(); - + public: void TestHash() { { @@ -54,7 +54,7 @@ public: TBlob temp = TBlob::FromStreamSingleThreaded(stream); TPlainHash<ui64, ui16> reader(temp); ui16 value = 0; - UNIT_ASSERT(reader.Find(5, &value)); + UNIT_ASSERT(reader.Find(5, &value)); UNIT_ASSERT_EQUAL(7, value); UNIT_ASSERT(!reader.Find(6, &value)); } @@ -99,7 +99,7 @@ public: UNIT_ASSERT_EQUAL(value, i); UNIT_ASSERT_EQUAL(reader.At(i), i); } - UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(ui32)); + UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(ui32)); } } { /// TString @@ -121,7 +121,7 @@ public: UNIT_ASSERT_EQUAL(value, ToString(i)); UNIT_ASSERT_EQUAL(reader.Get(i), ToString(i)); } - UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) * (N + 2) + N * 2); + UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) * (N + 2) + N * 2); } } { /// some other struct @@ -146,7 +146,7 @@ public: reader.Get(1, value); UNIT_ASSERT(value.x == 3 && value.y == 4.0); - UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) * (N + 2) + N * sizeof(TItem)); + UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) * (N + 2) + N * sizeof(TItem)); } } { /// pointer @@ -172,7 +172,7 @@ public: UNIT_ASSERT_EQUAL(value, a + i); UNIT_ASSERT_EQUAL(reader.At(i), a + i); } - UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(int*)); + UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(int*)); } } { /// std::pair<int, int> @@ -194,7 +194,7 @@ public: reader.Get(i, value); UNIT_ASSERT_EQUAL(value, TItem(i, i)); } - UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(TItem)); + UNIT_ASSERT_EQUAL(reader.RealSize(), sizeof(ui64) + N * sizeof(TItem)); } } } @@ -262,7 +262,7 @@ public: UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(1), ""); UNIT_ASSERT_STRINGS_EQUAL(reader.GetBlockName(2), "omega"); - UNIT_ASSERT_EQUAL(reader.GetBlockLenByName("alpha"), 6); // padding not included + UNIT_ASSERT_EQUAL(reader.GetBlockLenByName("alpha"), 6); // padding not included UNIT_ASSERT_EQUAL(reader.GetBlockLenByName("omega"), 17); UNIT_ASSERT(memcmp(reader.GetBlockByName("alpha"), "123456", 6) == 0); @@ -277,8 +277,8 @@ UNIT_TEST_SUITE_REGISTRATION(TChunkedHelpersTest); class TChunkedDataTest: public TTestBase { private: UNIT_TEST_SUITE(TChunkedDataTest); - UNIT_TEST(Test) - UNIT_TEST(TestEmpty) + UNIT_TEST(Test) + UNIT_TEST(TestEmpty) UNIT_TEST_SUITE_END(); void Test() { diff --git a/library/cpp/on_disk/chunks/reader.h b/library/cpp/on_disk/chunks/reader.h index c5fe783319..aba81a355f 100644 --- a/library/cpp/on_disk/chunks/reader.h +++ b/library/cpp/on_disk/chunks/reader.h @@ -7,51 +7,51 @@ class TBlob; class TChunkedDataReader { -public: - TChunkedDataReader(const TBlob& blob); +public: + TChunkedDataReader(const TBlob& blob); - inline const void* GetBlock(size_t index) const { - CheckIndex(index); - return Offsets[index]; - } + inline const void* GetBlock(size_t index) const { + CheckIndex(index); + return Offsets[index]; + } - inline size_t GetBlockLen(size_t index) const { - CheckIndex(index); + inline size_t GetBlockLen(size_t index) const { + CheckIndex(index); - if (Version == 0) { - if (index + 1 < Offsets.size()) { - return Offsets[index + 1] - Offsets[index]; + if (Version == 0) { + if (index + 1 < Offsets.size()) { + return Offsets[index + 1] - Offsets[index]; } - return Size - (Offsets.back() - Offsets.front()); + return Size - (Offsets.back() - Offsets.front()); } - return Lengths[index]; - } + return Lengths[index]; + } - TBlob GetBlob(size_t index) const; + TBlob GetBlob(size_t index) const; - template <typename T> + template <typename T> TArrayRef<const T> GetRegion(size_t index) const { - size_t len = GetBlockLen(index); - Y_ENSURE(len % sizeof(T) == 0, "wrong data padding"); + size_t len = GetBlockLen(index); + Y_ENSURE(len % sizeof(T) == 0, "wrong data padding"); return TArrayRef<const T>(reinterpret_cast<const T*>(GetBlock(index)), len / sizeof(T)); - } + } - inline size_t GetBlocksCount() const { - return Offsets.size(); - } + inline size_t GetBlocksCount() const { + return Offsets.size(); + } -private: - inline void CheckIndex(size_t index) const { - if (index >= GetBlocksCount()) { - ythrow yexception() << "requested block " << index << " of " << GetBlocksCount() << " blocks"; +private: + inline void CheckIndex(size_t index) const { + if (index >= GetBlocksCount()) { + ythrow yexception() << "requested block " << index << " of " << GetBlocksCount() << " blocks"; } - } + } -private: - ui64 Version = 0; - TVector<const char*> Offsets; - TVector<size_t> Lengths; - size_t Size = 0; +private: + ui64 Version = 0; + TVector<const char*> Offsets; + TVector<size_t> Lengths; + size_t Size = 0; }; diff --git a/library/cpp/on_disk/chunks/writer.h b/library/cpp/on_disk/chunks/writer.h index ab14522bdd..fbd02eef4b 100644 --- a/library/cpp/on_disk/chunks/writer.h +++ b/library/cpp/on_disk/chunks/writer.h @@ -9,49 +9,49 @@ inline void WriteBin(IOutputStream* out, typename TTypeTraits<T>::TFuncParam t) } class TChunkedDataWriter: public IOutputStream { -public: - TChunkedDataWriter(IOutputStream& slave); - ~TChunkedDataWriter() override; +public: + TChunkedDataWriter(IOutputStream& slave); + ~TChunkedDataWriter() override; - void NewBlock(); + void NewBlock(); - template <typename T> - inline void WriteBinary(typename TTypeTraits<T>::TFuncParam t) { - this->Write(&t, sizeof(T)); - } + template <typename T> + inline void WriteBinary(typename TTypeTraits<T>::TFuncParam t) { + this->Write(&t, sizeof(T)); + } - void WriteFooter(); + void WriteFooter(); - size_t GetCurrentBlockOffset() const; - size_t GetBlockCount() const; + size_t GetCurrentBlockOffset() const; + size_t GetBlockCount() const; -protected: - void DoWrite(const void* buf, size_t len) override { - Slave.Write(buf, len); - Offset += len; - } +protected: + void DoWrite(const void* buf, size_t len) override { + Slave.Write(buf, len); + Offset += len; + } -private: - static inline size_t PaddingSize(size_t size, size_t boundary) noexcept { - const size_t boundaryViolation = size % boundary; +private: + static inline size_t PaddingSize(size_t size, size_t boundary) noexcept { + const size_t boundaryViolation = size % boundary; - return boundaryViolation == 0 ? 0 : boundary - boundaryViolation; - } + return boundaryViolation == 0 ? 0 : boundary - boundaryViolation; + } - inline void Pad(size_t boundary) { - const size_t newOffset = Offset + PaddingSize(Offset, boundary); + inline void Pad(size_t boundary) { + const size_t newOffset = Offset + PaddingSize(Offset, boundary); - while (Offset < newOffset) { - Write('\0'); + while (Offset < newOffset) { + Write('\0'); } - } + } -private: - static const ui64 Version = 1; +private: + static const ui64 Version = 1; - IOutputStream& Slave; + IOutputStream& Slave; - size_t Offset; - TVector<ui64> Offsets; - TVector<ui64> Lengths; + size_t Offset; + TVector<ui64> Offsets; + TVector<ui64> Lengths; }; diff --git a/library/cpp/on_disk/chunks/ya.make b/library/cpp/on_disk/chunks/ya.make index acb52df5b0..93197cb1ce 100644 --- a/library/cpp/on_disk/chunks/ya.make +++ b/library/cpp/on_disk/chunks/ya.make @@ -1,7 +1,7 @@ LIBRARY() OWNER(g:util) - + SRCS( chunked_helpers.cpp reader.cpp diff --git a/library/cpp/openssl/init/init.cpp b/library/cpp/openssl/init/init.cpp index ae68ef08ea..a565c0ae5b 100644 --- a/library/cpp/openssl/init/init.cpp +++ b/library/cpp/openssl/init/init.cpp @@ -1,66 +1,66 @@ -#include "init.h" - -#include <util/generic/singleton.h> -#include <util/generic/vector.h> -#include <util/generic/ptr.h> -#include <util/generic/buffer.h> - -#include <util/system/yassert.h> -#include <util/system/mutex.h> -#include <util/system/thread.h> - -#include <util/random/entropy.h> -#include <util/stream/input.h> - -#include <openssl/bio.h> -#include <openssl/ssl.h> -#include <openssl/err.h> -#include <openssl/rand.h> -#include <openssl/conf.h> -#include <openssl/crypto.h> - -namespace { - struct TInitSsl { - struct TOpensslLocks { - inline TOpensslLocks() - : Mutexes(CRYPTO_num_locks()) - { - for (auto& mpref : Mutexes) { - mpref.Reset(new TMutex()); - } - } - - inline void LockOP(int mode, int n) { - auto& mutex = *Mutexes.at(n); - - if (mode & CRYPTO_LOCK) { - mutex.Acquire(); - } else { - mutex.Release(); - } - } - +#include "init.h" + +#include <util/generic/singleton.h> +#include <util/generic/vector.h> +#include <util/generic/ptr.h> +#include <util/generic/buffer.h> + +#include <util/system/yassert.h> +#include <util/system/mutex.h> +#include <util/system/thread.h> + +#include <util/random/entropy.h> +#include <util/stream/input.h> + +#include <openssl/bio.h> +#include <openssl/ssl.h> +#include <openssl/err.h> +#include <openssl/rand.h> +#include <openssl/conf.h> +#include <openssl/crypto.h> + +namespace { + struct TInitSsl { + struct TOpensslLocks { + inline TOpensslLocks() + : Mutexes(CRYPTO_num_locks()) + { + for (auto& mpref : Mutexes) { + mpref.Reset(new TMutex()); + } + } + + inline void LockOP(int mode, int n) { + auto& mutex = *Mutexes.at(n); + + if (mode & CRYPTO_LOCK) { + mutex.Acquire(); + } else { + mutex.Release(); + } + } + TVector<TAutoPtr<TMutex>> Mutexes; - }; - - inline TInitSsl() { + }; + + inline TInitSsl() { OPENSSL_init_crypto(OPENSSL_INIT_NO_ATEXIT, nullptr); - } - - inline ~TInitSsl() { + } + + inline ~TInitSsl() { OPENSSL_cleanup(); - } - - static void LockingFunction(int mode, int n, const char* /*file*/, int /*line*/) { - Singleton<TOpensslLocks>()->LockOP(mode, n); - } - - static unsigned long ThreadIdFunction() { - return TThread::CurrentThreadId(); - } - }; -} - -void InitOpenSSL() { + } + + static void LockingFunction(int mode, int n, const char* /*file*/, int /*line*/) { + Singleton<TOpensslLocks>()->LockOP(mode, n); + } + + static unsigned long ThreadIdFunction() { + return TThread::CurrentThreadId(); + } + }; +} + +void InitOpenSSL() { (void)SingletonWithPriority<TInitSsl, 0>(); -} +} diff --git a/library/cpp/openssl/init/init.h b/library/cpp/openssl/init/init.h index a626b316b3..699ce945ca 100644 --- a/library/cpp/openssl/init/init.h +++ b/library/cpp/openssl/init/init.h @@ -1,3 +1,3 @@ -#pragma once - -void InitOpenSSL(); +#pragma once + +void InitOpenSSL(); diff --git a/library/cpp/openssl/init/ya.make b/library/cpp/openssl/init/ya.make index aac073497c..a3eb56b6a0 100644 --- a/library/cpp/openssl/init/ya.make +++ b/library/cpp/openssl/init/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - +LIBRARY() + OWNER(pg g:zora) - -PEERDIR( - contrib/libs/openssl -) - -SRCS( - init.cpp -) - -END() + +PEERDIR( + contrib/libs/openssl +) + +SRCS( + init.cpp +) + +END() diff --git a/library/cpp/openssl/io/stream.cpp b/library/cpp/openssl/io/stream.cpp index 0b4be38c0e..eb6f9a9e5f 100644 --- a/library/cpp/openssl/io/stream.cpp +++ b/library/cpp/openssl/io/stream.cpp @@ -1,136 +1,136 @@ -#include "stream.h" - +#include "stream.h" + #include <util/generic/deque.h> -#include <util/generic/singleton.h> -#include <util/generic/yexception.h> - +#include <util/generic/singleton.h> +#include <util/generic/yexception.h> + #include <library/cpp/openssl/init/init.h> #include <library/cpp/openssl/method/io.h> #include <library/cpp/resource/resource.h> - -#include <openssl/bio.h> -#include <openssl/ssl.h> -#include <openssl/err.h> + +#include <openssl/bio.h> +#include <openssl/ssl.h> +#include <openssl/err.h> #include <openssl/tls1.h> #include <openssl/x509v3.h> - -using TOptions = TOpenSslClientIO::TOptions; - -namespace { - struct TSslIO; - - struct TSslInitOnDemand { - inline TSslInitOnDemand() { - InitOpenSSL(); - } - }; - + +using TOptions = TOpenSslClientIO::TOptions; + +namespace { + struct TSslIO; + + struct TSslInitOnDemand { + inline TSslInitOnDemand() { + InitOpenSSL(); + } + }; + int GetLastSslError() noexcept { - return ERR_peek_last_error(); - } - + return ERR_peek_last_error(); + } + const char* SslErrorText(int error) noexcept { return ERR_error_string(error, nullptr); - } - + } + inline TStringBuf SslLastError() noexcept { - return SslErrorText(GetLastSslError()); - } - - struct TSslError: public yexception { - inline TSslError() { - *this << SslLastError(); - } - }; - - struct TSslDestroy { + return SslErrorText(GetLastSslError()); + } + + struct TSslError: public yexception { + inline TSslError() { + *this << SslLastError(); + } + }; + + struct TSslDestroy { static inline void Destroy(ssl_ctx_st* ctx) noexcept { - SSL_CTX_free(ctx); - } - + SSL_CTX_free(ctx); + } + static inline void Destroy(ssl_st* ssl) noexcept { - SSL_free(ssl); - } - + SSL_free(ssl); + } + static inline void Destroy(bio_st* bio) noexcept { - BIO_free(bio); - } + BIO_free(bio); + } static inline void Destroy(x509_st* x509) noexcept { X509_free(x509); } - }; - - template <class T> + }; + + template <class T> using TSslHolderPtr = THolder<T, TSslDestroy>; - + using TSslContextPtr = TSslHolderPtr<ssl_ctx_st>; using TSslPtr = TSslHolderPtr<ssl_st>; using TBioPtr = TSslHolderPtr<bio_st>; using TX509Ptr = TSslHolderPtr<x509_st>; - + inline TSslContextPtr CreateSslCtx(const ssl_method_st* method) { - TSslContextPtr ctx(SSL_CTX_new(method)); - - if (!ctx) { - ythrow TSslError() << "SSL_CTX_new"; - } - - SSL_CTX_set_options(ctx.Get(), SSL_OP_NO_SSLv2); + TSslContextPtr ctx(SSL_CTX_new(method)); + + if (!ctx) { + ythrow TSslError() << "SSL_CTX_new"; + } + + SSL_CTX_set_options(ctx.Get(), SSL_OP_NO_SSLv2); SSL_CTX_set_options(ctx.Get(), SSL_OP_NO_SSLv3); - SSL_CTX_set_options(ctx.Get(), SSL_OP_MICROSOFT_SESS_ID_BUG); - SSL_CTX_set_options(ctx.Get(), SSL_OP_NETSCAPE_CHALLENGE_BUG); - - return ctx; - } - + SSL_CTX_set_options(ctx.Get(), SSL_OP_MICROSOFT_SESS_ID_BUG); + SSL_CTX_set_options(ctx.Get(), SSL_OP_NETSCAPE_CHALLENGE_BUG); + + return ctx; + } + struct TStreamIO : public NOpenSSL::TAbstractIO { inline TStreamIO(IInputStream* in, IOutputStream* out) : In(in) , Out(out) { } - + int Write(const char* data, size_t dlen, size_t* written) override { Out->Write(data, dlen); *written = dlen; return 1; - } - + } + int Read(char* data, size_t dlen, size_t* readbytes) override { *readbytes = In->Read(data, dlen); return 1; - } - + } + int Puts(const char* buf) override { Y_UNUSED(buf); return -1; - } - + } + int Gets(char* buf, int size) override { Y_UNUSED(buf); Y_UNUSED(size); return -1; - } - + } + void Flush() override { - } - + } + IInputStream* In; IOutputStream* Out; - }; - - struct TSslIO: public TSslInitOnDemand, public TOptions { + }; + + struct TSslIO: public TSslInitOnDemand, public TOptions { inline TSslIO(IInputStream* in, IOutputStream* out, const TOptions& opts) - : TOptions(opts) + : TOptions(opts) , Io(in, out) - , Ctx(CreateClientContext()) - , Ssl(ConstructSsl()) - { - Connect(); - } - + , Ctx(CreateClientContext()) + , Ssl(ConstructSsl()) + { + Connect(); + } + inline TSslContextPtr CreateClientContext() { TSslContextPtr ctx = CreateSslCtx(SSLv23_client_method()); if (ClientCert_) { @@ -164,23 +164,23 @@ namespace { return ctx; } - inline TSslPtr ConstructSsl() { - TSslPtr ssl(SSL_new(Ctx.Get())); - - if (!ssl) { - ythrow TSslError() << "SSL_new"; - } - + inline TSslPtr ConstructSsl() { + TSslPtr ssl(SSL_new(Ctx.Get())); + + if (!ssl) { + ythrow TSslError() << "SSL_new"; + } + if (VerifyCert_) { InitVerification(ssl.Get()); } BIO_up_ref(Io); // SSL_set_bio consumes only one reference if rbio and wbio are the same SSL_set_bio(ssl.Get(), Io, Io); - - return ssl; - } - + + return ssl; + } + inline void InitVerification(ssl_st* ssl) { X509_VERIFY_PARAM* param = SSL_get0_param(ssl); X509_VERIFY_PARAM_set_hostflags(param, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); @@ -200,76 +200,76 @@ namespace { SSL_set_verify(ssl, SSL_VERIFY_PEER, nullptr); } - inline void Connect() { - if (SSL_connect(Ssl.Get()) != 1) { - ythrow TSslError() << "SSL_connect"; - } - } - - inline void Finish() const { - SSL_shutdown(Ssl.Get()); - } - - inline size_t Read(void* buf, size_t len) { - const int ret = SSL_read(Ssl.Get(), buf, len); - - if (ret < 0) { - ythrow TSslError() << "SSL_read"; - } - - return ret; - } - - inline void Write(const char* buf, size_t len) { - while (len) { - const int ret = SSL_write(Ssl.Get(), buf, len); - - if (ret < 0) { - ythrow TSslError() << "SSL_write"; - } - - buf += (size_t)ret; - len -= (size_t)ret; - } - } - + inline void Connect() { + if (SSL_connect(Ssl.Get()) != 1) { + ythrow TSslError() << "SSL_connect"; + } + } + + inline void Finish() const { + SSL_shutdown(Ssl.Get()); + } + + inline size_t Read(void* buf, size_t len) { + const int ret = SSL_read(Ssl.Get(), buf, len); + + if (ret < 0) { + ythrow TSslError() << "SSL_read"; + } + + return ret; + } + + inline void Write(const char* buf, size_t len) { + while (len) { + const int ret = SSL_write(Ssl.Get(), buf, len); + + if (ret < 0) { + ythrow TSslError() << "SSL_write"; + } + + buf += (size_t)ret; + len -= (size_t)ret; + } + } + TStreamIO Io; - TSslContextPtr Ctx; - TSslPtr Ssl; - }; -} - -struct TOpenSslClientIO::TImpl: public TSslIO { + TSslContextPtr Ctx; + TSslPtr Ssl; + }; +} + +struct TOpenSslClientIO::TImpl: public TSslIO { inline TImpl(IInputStream* in, IOutputStream* out, const TOptions& opts) - : TSslIO(in, out, opts) - { - } -}; - + : TSslIO(in, out, opts) + { + } +}; + TOpenSslClientIO::TOpenSslClientIO(IInputStream* in, IOutputStream* out) - : Impl_(new TImpl(in, out, TOptions())) -{ -} - + : Impl_(new TImpl(in, out, TOptions())) +{ +} + TOpenSslClientIO::TOpenSslClientIO(IInputStream* in, IOutputStream* out, const TOptions& options) - : Impl_(new TImpl(in, out, options)) -{ -} - -TOpenSslClientIO::~TOpenSslClientIO() { - try { - Impl_->Finish(); - } catch (...) { - } -} - -void TOpenSslClientIO::DoWrite(const void* buf, size_t len) { - Impl_->Write((const char*)buf, len); -} - -size_t TOpenSslClientIO::DoRead(void* buf, size_t len) { - return Impl_->Read(buf, len); -} + : Impl_(new TImpl(in, out, options)) +{ +} + +TOpenSslClientIO::~TOpenSslClientIO() { + try { + Impl_->Finish(); + } catch (...) { + } +} + +void TOpenSslClientIO::DoWrite(const void* buf, size_t len) { + Impl_->Write((const char*)buf, len); +} + +size_t TOpenSslClientIO::DoRead(void* buf, size_t len) { + return Impl_->Read(buf, len); +} namespace NPrivate { void TSslDestroy::Destroy(x509_store_st* x509) noexcept { diff --git a/library/cpp/openssl/io/stream.h b/library/cpp/openssl/io/stream.h index 7bca8f80ef..be1837618c 100644 --- a/library/cpp/openssl/io/stream.h +++ b/library/cpp/openssl/io/stream.h @@ -1,13 +1,13 @@ -#pragma once - +#pragma once + #include <util/generic/maybe.h> -#include <util/generic/ptr.h> +#include <util/generic/ptr.h> #include <util/stream/input.h> #include <util/stream/output.h> - + class TOpenSslClientIO: public IInputStream, public IOutputStream { -public: - struct TOptions { +public: + struct TOptions { struct TVerifyCert { // Uses builtin certs. // Also uses default CA path /etc/ssl/certs/ - can be provided with debian package: ca-certificates.deb. @@ -22,21 +22,21 @@ public: TMaybe<TVerifyCert> VerifyCert_; TMaybe<TClientCert> ClientCert_; - // TODO - keys, cyphers, etc - }; - + // TODO - keys, cyphers, etc + }; + TOpenSslClientIO(IInputStream* in, IOutputStream* out); TOpenSslClientIO(IInputStream* in, IOutputStream* out, const TOptions& options); ~TOpenSslClientIO() override; - -private: + +private: void DoWrite(const void* buf, size_t len) override; size_t DoRead(void* buf, size_t len) override; - -private: - struct TImpl; - THolder<TImpl> Impl_; -}; + +private: + struct TImpl; + THolder<TImpl> Impl_; +}; struct x509_store_st; diff --git a/library/cpp/openssl/io/ya.make b/library/cpp/openssl/io/ya.make index aaebba4011..10b71d4177 100644 --- a/library/cpp/openssl/io/ya.make +++ b/library/cpp/openssl/io/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - -OWNER(pg) - -PEERDIR( +LIBRARY() + +OWNER(pg) + +PEERDIR( certs - contrib/libs/openssl + contrib/libs/openssl library/cpp/openssl/init library/cpp/openssl/method -) - -SRCS( - stream.cpp -) - -END() +) + +SRCS( + stream.cpp +) + +END() diff --git a/library/cpp/openssl/ya.make b/library/cpp/openssl/ya.make index 7c10963e26..80b943e360 100644 --- a/library/cpp/openssl/ya.make +++ b/library/cpp/openssl/ya.make @@ -1,13 +1,13 @@ -RECURSE( - big_integer - big_integer/ut +RECURSE( + big_integer + big_integer/ut crypto crypto/ut holders holders/ut - io + io io/ut method method/ut - init -) + init +) diff --git a/library/cpp/packedtypes/fixed_point.h b/library/cpp/packedtypes/fixed_point.h index b36b80876e..63b45a73c5 100644 --- a/library/cpp/packedtypes/fixed_point.h +++ b/library/cpp/packedtypes/fixed_point.h @@ -1,71 +1,71 @@ #pragma once namespace NFixedPoint { - template <ui64 FracMult> - class TFixedPoint { - typedef TFixedPoint<FracMult> TSelf; - ui64 Rep; + template <ui64 FracMult> + class TFixedPoint { + typedef TFixedPoint<FracMult> TSelf; + ui64 Rep; - public: - TFixedPoint() - : Rep() - { - } + public: + TFixedPoint() + : Rep() + { + } - template <typename T> - explicit TFixedPoint(T t) - : Rep(ui64(t * FracMult)) - { - } + template <typename T> + explicit TFixedPoint(T t) + : Rep(ui64(t * FracMult)) + { + } - explicit TFixedPoint(ui64 i, ui64 f) - : Rep(i * FracMult + f % FracMult) - { - } + explicit TFixedPoint(ui64 i, ui64 f) + : Rep(i * FracMult + f % FracMult) + { + } - template <typename T> - TSelf& operator=(T t) { - Rep = t * FracMult; - return *this; - } + template <typename T> + TSelf& operator=(T t) { + Rep = t * FracMult; + return *this; + } - operator double() const { - return Int() + double(Frac()) / FracMult; - } + operator double() const { + return Int() + double(Frac()) / FracMult; + } - operator ui64() const { - return Int(); - } + operator ui64() const { + return Int(); + } - template <typename T> - TSelf& operator/=(T t) { - Rep = ui64(Rep / t); - return *this; - } + template <typename T> + TSelf& operator/=(T t) { + Rep = ui64(Rep / t); + return *this; + } - template <typename T> - TSelf operator/(T t) const { - TSelf r = *this; - return r /= t; - } + template <typename T> + TSelf operator/(T t) const { + TSelf r = *this; + return r /= t; + } - ui64 Frac() const { - return Rep % FracMult; - } + ui64 Frac() const { + return Rep % FracMult; + } - ui64 Int() const { - return Rep / FracMult; - } + ui64 Int() const { + return Rep / FracMult; + } - static ui64 Mult() { - return FracMult; - } + static ui64 Mult() { + return FracMult; + } - static TSelf Make(ui64 rep) { - TFixedPoint<FracMult> fp; - fp.Rep = rep; - return fp; - } - }; + static TSelf Make(ui64 rep) { + TFixedPoint<FracMult> fp; + fp.Rep = rep; + return fp; + } + }; } diff --git a/library/cpp/packedtypes/longs.cpp b/library/cpp/packedtypes/longs.cpp index 4cb6f4963b..162322af44 100644 --- a/library/cpp/packedtypes/longs.cpp +++ b/library/cpp/packedtypes/longs.cpp @@ -1 +1 @@ -#include "longs.h" +#include "longs.h" diff --git a/library/cpp/packedtypes/longs.h b/library/cpp/packedtypes/longs.h index 084098d705..ddf2a9d3ca 100644 --- a/library/cpp/packedtypes/longs.h +++ b/library/cpp/packedtypes/longs.h @@ -1,45 +1,45 @@ #pragma once -#include <util/system/defaults.h> // _BIDSCLASS _EXPCLASS +#include <util/system/defaults.h> // _BIDSCLASS _EXPCLASS #include <util/system/yassert.h> #include <util/system/unaligned_mem.h> + +#define PUT_8(x, buf, shift) WriteUnaligned<ui8>((buf)++, (x) >> (shift)) +#define GET_8_OR(x, buf, type, shift) (x) |= (type) * (buf)++ << (shift) -#define PUT_8(x, buf, shift) WriteUnaligned<ui8>((buf)++, (x) >> (shift)) -#define GET_8_OR(x, buf, type, shift) (x) |= (type) * (buf)++ << (shift) - -#if defined(_big_endian_) +#if defined(_big_endian_) #define LO_SHIFT 1 #define HI_SHIFT 0 -#elif defined(_little_endian_) +#elif defined(_little_endian_) #define LO_SHIFT 0 #define HI_SHIFT 1 #endif -#if !defined(_must_align2_) -#define PUT_16(x, buf, shift) WriteUnaligned<ui16>(buf, (x) >> (shift)), (buf) += 2 -#define GET_16_OR(x, buf, type, shift) (x) |= (type)ReadUnaligned<ui16>(buf) << (shift), (buf) += 2 +#if !defined(_must_align2_) +#define PUT_16(x, buf, shift) WriteUnaligned<ui16>(buf, (x) >> (shift)), (buf) += 2 +#define GET_16_OR(x, buf, type, shift) (x) |= (type)ReadUnaligned<ui16>(buf) << (shift), (buf) += 2 #else -#define PUT_16(x, buf, shift) PUT_8(x, buf, shift + 8 * LO_SHIFT), PUT_8(x, buf, shift + 8 * HI_SHIFT) -#define GET_16_OR(x, buf, type, shift) GET_8_OR(x, buf, type, shift + 8 * LO_SHIFT), GET_8_OR(x, buf, type, shift + 8 * HI_SHIFT) +#define PUT_16(x, buf, shift) PUT_8(x, buf, shift + 8 * LO_SHIFT), PUT_8(x, buf, shift + 8 * HI_SHIFT) +#define GET_16_OR(x, buf, type, shift) GET_8_OR(x, buf, type, shift + 8 * LO_SHIFT), GET_8_OR(x, buf, type, shift + 8 * HI_SHIFT) #endif -#if !defined(_must_align4_) -#define PUT_32(x, buf, shift) WriteUnaligned<ui32>(buf, (x) >> (shift)), (buf) += 4 -#define GET_32_OR(x, buf, type, shift) (x) |= (type)ReadUnaligned<ui32>(buf) << (shift), (buf) += 4 +#if !defined(_must_align4_) +#define PUT_32(x, buf, shift) WriteUnaligned<ui32>(buf, (x) >> (shift)), (buf) += 4 +#define GET_32_OR(x, buf, type, shift) (x) |= (type)ReadUnaligned<ui32>(buf) << (shift), (buf) += 4 #else -#define PUT_32(x, buf, shift) PUT_16(x, buf, shift + 16 * LO_SHIFT), PUT_16(x, buf, shift + 16 * HI_SHIFT) -#define GET_32_OR(x, buf, type, shift) GET_16_OR(x, buf, type, shift + 16 * LO_SHIFT), GET_16_OR(x, buf, type, shift + 16 * HI_SHIFT) +#define PUT_32(x, buf, shift) PUT_16(x, buf, shift + 16 * LO_SHIFT), PUT_16(x, buf, shift + 16 * HI_SHIFT) +#define GET_32_OR(x, buf, type, shift) GET_16_OR(x, buf, type, shift + 16 * LO_SHIFT), GET_16_OR(x, buf, type, shift + 16 * HI_SHIFT) #endif -#if !defined(_must_align8_) -#define PUT_64(x, buf, shift) WriteUnaligned<ui64>(buf, (x) >> (shift)), (buf) += 8 -#define GET_64_OR(x, buf, type, shift) (x) |= (type)ReadUnaligned<ui64>(buf) << (shift), (buf) += 8 +#if !defined(_must_align8_) +#define PUT_64(x, buf, shift) WriteUnaligned<ui64>(buf, (x) >> (shift)), (buf) += 8 +#define GET_64_OR(x, buf, type, shift) (x) |= (type)ReadUnaligned<ui64>(buf) << (shift), (buf) += 8 #else -#define PUT_64(x, buf, shift) PUT_32(x, buf, shift + 32 * LO_SHIFT), PUT_32(x, buf, shift + 32 * HI_SHIFT) -#define GET_64_OR(x, buf, type, shift) GET_32_OR(x, buf, type, shift + 32 * LO_SHIFT), GET_32_OR(x, buf, type, shift + 32 * HI_SHIFT) +#define PUT_64(x, buf, shift) PUT_32(x, buf, shift + 32 * LO_SHIFT), PUT_32(x, buf, shift + 32 * HI_SHIFT) +#define GET_64_OR(x, buf, type, shift) GET_32_OR(x, buf, type, shift + 32 * LO_SHIFT), GET_32_OR(x, buf, type, shift + 32 * HI_SHIFT) #endif -struct mem_traits { +struct mem_traits { static ui8 get_8(const char*& mem) { ui8 x = 0; GET_8_OR(x, mem, ui8, 0); @@ -64,7 +64,7 @@ struct mem_traits { static void put_32(ui32 x, char*& mem) { PUT_32(x, mem, 0); } - static int is_good(char*&) { + static int is_good(char*&) { return 1; } }; @@ -93,216 +93,216 @@ struct mem_traits { #define MY_28(x) ((ui32)(x) < PACK2LIM ? MY_14(x) : ((ui32)(x) < PACK3LIM ? 3 : 4)) #define MY_32(x) ((ui32)(x) < PACK4LIM ? MY_28(x) : 5) -#define MY_64(x) ((ui64)(x) < PACK4LIM ? MY_28(x) : ((ui64)(x) < PACK6LIM ? ((ui64)(x) < PACK5LIM ? 5 : 6) : ((ui64)(x) < PACK7LIM ? 7 : ((ui64)(x) < PACK8LIM ? 8 : 9)))) +#define MY_64(x) ((ui64)(x) < PACK4LIM ? MY_28(x) : ((ui64)(x) < PACK6LIM ? ((ui64)(x) < PACK5LIM ? 5 : 6) : ((ui64)(x) < PACK7LIM ? 7 : ((ui64)(x) < PACK8LIM ? 8 : 9)))) -#if !defined(MACRO_BEGIN) -#define MACRO_BEGIN do { -#define MACRO_END \ - } \ - while (0) +#if !defined(MACRO_BEGIN) +#define MACRO_BEGIN do { +#define MACRO_END \ + } \ + while (0) #endif -#define PACK_14(x, buf, how, ret) \ - MACRO_BEGIN \ - if ((ui16)(x) < PACK1LIM) { \ - how::put_8((ui8)(x), (buf)); \ - (ret) = 1; \ - } else { \ - how::put_8((ui8)(0x80 | (x) >> 8), (buf)); \ - how::put_8((ui8)(x), (buf)); \ - (ret) = 2; \ - } \ - MACRO_END - -#define PACK_28(x, buf, how, ret) \ - MACRO_BEGIN \ - if ((ui32)(x) < PACK2LIM) { \ - PACK_14(x, buf, how, ret); \ - } else { \ - if ((ui32)(x) < PACK3LIM) { \ - how::put_8((ui8)(0xC0 | (x) >> 16), buf); \ - (ret) = 3; \ - } else { \ - how::put_8((ui8)(0xE0 | (x) >> 24), buf); \ - how::put_8((ui8)((x) >> 16), buf); \ - (ret) = 4; \ - } \ - how::put_16((ui16)(x), (buf)); \ - } \ - MACRO_END - -#define PACK_32(x, buf, how, ret) \ - MACRO_BEGIN \ - if ((ui32)(x) < PACK4LIM) { \ - PACK_28(x, buf, how, ret); \ - } else { \ - how::put_8((ui8)(0xF0), buf); \ - how::put_32((ui32)(x), buf); \ - (ret) = 5; \ - } \ - MACRO_END - -#define PACK_64(x, buf, how, ret) \ - MACRO_BEGIN \ - if ((ui64)(x) < PACK4LIM) { \ - PACK_28((ui32)(x), buf, how, ret); \ - } else { \ - if ((ui64)(x) < PACK6LIM) { \ - if ((ui64)(x) < PACK5LIM) { \ - how::put_8((ui8)(0xF0 | (x) >> 32), buf); \ - (ret) = 5; \ - } else { \ - how::put_8((ui8)(0xF8 | (x) >> 40), buf); \ - how::put_8((ui8)((x) >> 32), buf); \ - (ret) = 6; \ - } \ - } else { \ - if ((ui64)(x) < PACK7LIM) { \ - how::put_8((ui8)(0xFC | (x) >> 48), buf); \ - (ret) = 7; \ - } else { \ - if ((ui64)(x) < PACK8LIM) { \ - how::put_8((ui8)(0xFE | (x) >> 56), buf); \ - how::put_8((ui8)((x) >> 48), buf); \ - (ret) = 8; \ - } else { \ - how::put_8((ui8)(0xFF), buf); \ - how::put_16((ui16)((x) >> 48), buf); \ - (ret) = 9; \ - } \ - } \ - how::put_16((ui16)((x) >> 32), buf); \ - } \ - how::put_32((ui32)(x), buf); \ - } \ - MACRO_END - -#define DO_UNPACK_14(firstByte, x, buf, how, ret) \ - MACRO_BEGIN \ - if (firstByte < 0x80) { \ - (x) = (firstByte); \ - (ret) = 1; \ - } else { \ - (x) = (firstByte & 0x7F) << 8; \ - (x) |= how::get_8(buf); \ - (ret) = 2; \ - } \ - MACRO_END - -#define UNPACK_14(x, buf, how, ret) \ - MACRO_BEGIN \ - ui8 firstByte = how::get_8(buf); \ - DO_UNPACK_14(firstByte, x, buf, how, ret); \ - MACRO_END - -#define DO_UNPACK_28(firstByte, x, buf, how, ret) \ - MACRO_BEGIN \ - if (firstByte < 0xC0) { \ - DO_UNPACK_14(firstByte, x, buf, how, ret); \ - } else { \ - if (firstByte < 0xE0) { \ - (x) = (firstByte & 0x3F) << 16; \ - (ret) = 3; \ - } else { \ - (x) = (firstByte & 0x1F) << 24; \ - (x) |= how::get_8(buf) << 16; \ - (ret) = 4; \ - } \ - (x) |= how::get_16(buf); \ - } \ - MACRO_END - -#define UNPACK_28(x, buf, how, ret) \ - MACRO_BEGIN \ - ui8 firstByte = how::get_8(buf); \ - DO_UNPACK_28(firstByte, x, buf, how, ret); \ - MACRO_END - -#define DO_UNPACK_32(firstByte, x, buf, how, ret) \ - MACRO_BEGIN \ - if (firstByte < 0xF0) { \ - DO_UNPACK_28(firstByte, x, buf, how, ret); \ - } else { \ - (x) = how::get_32(buf); \ - (ret) = 5; \ - } \ - MACRO_END - -#define UNPACK_32(x, buf, how, ret) \ - MACRO_BEGIN \ - ui8 firstByte = how::get_8(buf); \ - DO_UNPACK_32(firstByte, x, buf, how, ret); \ - MACRO_END - -#define DO_UNPACK_64(firstByte, x, buf, how, ret) \ - MACRO_BEGIN \ - if (firstByte < 0xF0) { \ - DO_UNPACK_28(firstByte, x, buf, how, ret); \ - } else { \ - if (firstByte < 0xFC) { \ - if (firstByte < 0xF8) { \ - (x) = (ui64)(firstByte & 0x0F) << 32; \ - (ret) = 5; \ - } else { \ - (x) = (ui64)(firstByte & 0x07) << 40; \ - (x) |= (ui64)how::get_8(buf) << 32; \ - (ret) = 6; \ - } \ - } else { \ - if (firstByte < 0xFE) { \ - (x) = (ui64)(firstByte & 0x03) << 48; \ - (ret) = 7; \ - } else { \ - if (firstByte < 0xFF) { \ - (x) = (ui64)(firstByte & 0x01) << 56; \ - (x) |= (ui64)how::get_8(buf) << 48; \ - (ret) = 8; \ - } else { \ - (x) = (ui64)how::get_16(buf) << 48; \ - (ret) = 9; \ - } \ - } \ - (x) |= (ui64)how::get_16(buf) << 32; \ - } \ - (x) |= how::get_32(buf); \ - } \ - MACRO_END - -#define UNPACK_64(x, buf, how, ret) \ - MACRO_BEGIN \ - ui8 firstByte = how::get_8(buf); \ - DO_UNPACK_64(firstByte, x, buf, how, ret); \ - MACRO_END - -inline int in_long(i64& longVal, const char* ptrBuf) { +#define PACK_14(x, buf, how, ret) \ + MACRO_BEGIN \ + if ((ui16)(x) < PACK1LIM) { \ + how::put_8((ui8)(x), (buf)); \ + (ret) = 1; \ + } else { \ + how::put_8((ui8)(0x80 | (x) >> 8), (buf)); \ + how::put_8((ui8)(x), (buf)); \ + (ret) = 2; \ + } \ + MACRO_END + +#define PACK_28(x, buf, how, ret) \ + MACRO_BEGIN \ + if ((ui32)(x) < PACK2LIM) { \ + PACK_14(x, buf, how, ret); \ + } else { \ + if ((ui32)(x) < PACK3LIM) { \ + how::put_8((ui8)(0xC0 | (x) >> 16), buf); \ + (ret) = 3; \ + } else { \ + how::put_8((ui8)(0xE0 | (x) >> 24), buf); \ + how::put_8((ui8)((x) >> 16), buf); \ + (ret) = 4; \ + } \ + how::put_16((ui16)(x), (buf)); \ + } \ + MACRO_END + +#define PACK_32(x, buf, how, ret) \ + MACRO_BEGIN \ + if ((ui32)(x) < PACK4LIM) { \ + PACK_28(x, buf, how, ret); \ + } else { \ + how::put_8((ui8)(0xF0), buf); \ + how::put_32((ui32)(x), buf); \ + (ret) = 5; \ + } \ + MACRO_END + +#define PACK_64(x, buf, how, ret) \ + MACRO_BEGIN \ + if ((ui64)(x) < PACK4LIM) { \ + PACK_28((ui32)(x), buf, how, ret); \ + } else { \ + if ((ui64)(x) < PACK6LIM) { \ + if ((ui64)(x) < PACK5LIM) { \ + how::put_8((ui8)(0xF0 | (x) >> 32), buf); \ + (ret) = 5; \ + } else { \ + how::put_8((ui8)(0xF8 | (x) >> 40), buf); \ + how::put_8((ui8)((x) >> 32), buf); \ + (ret) = 6; \ + } \ + } else { \ + if ((ui64)(x) < PACK7LIM) { \ + how::put_8((ui8)(0xFC | (x) >> 48), buf); \ + (ret) = 7; \ + } else { \ + if ((ui64)(x) < PACK8LIM) { \ + how::put_8((ui8)(0xFE | (x) >> 56), buf); \ + how::put_8((ui8)((x) >> 48), buf); \ + (ret) = 8; \ + } else { \ + how::put_8((ui8)(0xFF), buf); \ + how::put_16((ui16)((x) >> 48), buf); \ + (ret) = 9; \ + } \ + } \ + how::put_16((ui16)((x) >> 32), buf); \ + } \ + how::put_32((ui32)(x), buf); \ + } \ + MACRO_END + +#define DO_UNPACK_14(firstByte, x, buf, how, ret) \ + MACRO_BEGIN \ + if (firstByte < 0x80) { \ + (x) = (firstByte); \ + (ret) = 1; \ + } else { \ + (x) = (firstByte & 0x7F) << 8; \ + (x) |= how::get_8(buf); \ + (ret) = 2; \ + } \ + MACRO_END + +#define UNPACK_14(x, buf, how, ret) \ + MACRO_BEGIN \ + ui8 firstByte = how::get_8(buf); \ + DO_UNPACK_14(firstByte, x, buf, how, ret); \ + MACRO_END + +#define DO_UNPACK_28(firstByte, x, buf, how, ret) \ + MACRO_BEGIN \ + if (firstByte < 0xC0) { \ + DO_UNPACK_14(firstByte, x, buf, how, ret); \ + } else { \ + if (firstByte < 0xE0) { \ + (x) = (firstByte & 0x3F) << 16; \ + (ret) = 3; \ + } else { \ + (x) = (firstByte & 0x1F) << 24; \ + (x) |= how::get_8(buf) << 16; \ + (ret) = 4; \ + } \ + (x) |= how::get_16(buf); \ + } \ + MACRO_END + +#define UNPACK_28(x, buf, how, ret) \ + MACRO_BEGIN \ + ui8 firstByte = how::get_8(buf); \ + DO_UNPACK_28(firstByte, x, buf, how, ret); \ + MACRO_END + +#define DO_UNPACK_32(firstByte, x, buf, how, ret) \ + MACRO_BEGIN \ + if (firstByte < 0xF0) { \ + DO_UNPACK_28(firstByte, x, buf, how, ret); \ + } else { \ + (x) = how::get_32(buf); \ + (ret) = 5; \ + } \ + MACRO_END + +#define UNPACK_32(x, buf, how, ret) \ + MACRO_BEGIN \ + ui8 firstByte = how::get_8(buf); \ + DO_UNPACK_32(firstByte, x, buf, how, ret); \ + MACRO_END + +#define DO_UNPACK_64(firstByte, x, buf, how, ret) \ + MACRO_BEGIN \ + if (firstByte < 0xF0) { \ + DO_UNPACK_28(firstByte, x, buf, how, ret); \ + } else { \ + if (firstByte < 0xFC) { \ + if (firstByte < 0xF8) { \ + (x) = (ui64)(firstByte & 0x0F) << 32; \ + (ret) = 5; \ + } else { \ + (x) = (ui64)(firstByte & 0x07) << 40; \ + (x) |= (ui64)how::get_8(buf) << 32; \ + (ret) = 6; \ + } \ + } else { \ + if (firstByte < 0xFE) { \ + (x) = (ui64)(firstByte & 0x03) << 48; \ + (ret) = 7; \ + } else { \ + if (firstByte < 0xFF) { \ + (x) = (ui64)(firstByte & 0x01) << 56; \ + (x) |= (ui64)how::get_8(buf) << 48; \ + (ret) = 8; \ + } else { \ + (x) = (ui64)how::get_16(buf) << 48; \ + (ret) = 9; \ + } \ + } \ + (x) |= (ui64)how::get_16(buf) << 32; \ + } \ + (x) |= how::get_32(buf); \ + } \ + MACRO_END + +#define UNPACK_64(x, buf, how, ret) \ + MACRO_BEGIN \ + ui8 firstByte = how::get_8(buf); \ + DO_UNPACK_64(firstByte, x, buf, how, ret); \ + MACRO_END + +inline int in_long(i64& longVal, const char* ptrBuf) { int ret; UNPACK_64(longVal, ptrBuf, mem_traits, ret); return ret; } -inline int out_long(const i64& longVal, char* ptrBuf) { +inline int out_long(const i64& longVal, char* ptrBuf) { int ret; - PACK_64(longVal, ptrBuf, mem_traits, ret); /*7*/ + PACK_64(longVal, ptrBuf, mem_traits, ret); /*7*/ return ret; } -inline int len_long(const i64& longVal) { +inline int len_long(const i64& longVal) { return MY_64(longVal); } -inline int in_long(i32& longVal, const char* ptrBuf) { +inline int in_long(i32& longVal, const char* ptrBuf) { int ret; UNPACK_32(longVal, ptrBuf, mem_traits, ret); return ret; } -inline int out_long(const i32& longVal, char* ptrBuf) { +inline int out_long(const i32& longVal, char* ptrBuf) { int ret; PACK_32(longVal, ptrBuf, mem_traits, ret); return ret; } -inline int len_long(const i32& longVal) { +inline int len_long(const i32& longVal) { return MY_32(longVal); } diff --git a/library/cpp/packedtypes/longs_ut.cpp b/library/cpp/packedtypes/longs_ut.cpp index 8b06c934d2..e09c03d8ac 100644 --- a/library/cpp/packedtypes/longs_ut.cpp +++ b/library/cpp/packedtypes/longs_ut.cpp @@ -1,9 +1,9 @@ -#include "longs.h" - +#include "longs.h" + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/digest/old_crc/crc.h> -#include <util/string/util.h> +#include <util/string/util.h> #include <util/stream/output.h> #include <util/system/hi_lo.h> @@ -19,7 +19,7 @@ Y_UNIT_TEST_SUITE(TLongsTest) { char buf[100]; memset(buf, 0, 100); - char* p = buf; + char* p = buf; int l = out_long(x64, buf); s += Sprintf("x64=0x%" PRIi64 "\n", x64); s += Sprintf("LO_32(x64)=0x%" PRIu32 " HI_32(x64)=0x%" PRIu32 "\n", (ui32)Lo32(x64), (ui32)Hi32(x64)); @@ -28,7 +28,7 @@ Y_UNIT_TEST_SUITE(TLongsTest) { s += Sprintf("0x%02x ", buf[i]); } s += Sprintf("\n"); - + p = buf; in_long(y64, p); s += Sprintf("x=0x%" PRIi64 " y=0x%" PRIi64 "\n", x64, y64); @@ -37,7 +37,7 @@ Y_UNIT_TEST_SUITE(TLongsTest) { } else { s += Sprintf("OK\n"); } - + UNIT_ASSERT_EQUAL(Crc<ui64>(s.data(), s.size()), 7251624297500315779ULL); // WTF? } @@ -58,9 +58,9 @@ Y_UNIT_TEST_SUITE(TLongsTest) { TestOneValue<TSignedInt>((TSignedInt)(1ull << i)); TestOneValue<TSignedInt>((TSignedInt)((1ull << i) - 1)); TestOneValue<TSignedInt>((TSignedInt)((1ull << i) + 1)); - } + } } - + Y_UNIT_TEST(TestCornerCases) { TestCornerCasesImpl<i32>(31); TestCornerCasesImpl<i64>(63); diff --git a/library/cpp/packedtypes/packed.h b/library/cpp/packedtypes/packed.h index 88cff26ae2..340fd99472 100644 --- a/library/cpp/packedtypes/packed.h +++ b/library/cpp/packedtypes/packed.h @@ -7,8 +7,8 @@ #include "longs.h" -struct Stream_traits { - template <typename T> +struct Stream_traits { + template <typename T> static T get(IInputStream& in) { T x; ::Load(&in, x); @@ -32,23 +32,23 @@ struct Stream_traits { static void put_32(ui32 x, IOutputStream& out) { ::Save(&out, x); } - static int is_good(IInputStream& /*in*/) { - return 1; - } - static int is_good(IOutputStream& /*out*/) { - return 1; - } + static int is_good(IInputStream& /*in*/) { + return 1; + } + static int is_good(IOutputStream& /*out*/) { + return 1; + } }; -struct TZCMemoryInput_traits { - template <typename T> +struct TZCMemoryInput_traits { + template <typename T> static T get(TZCMemoryInput& in) { T x; in.ReadPOD(x); return x; } - static ui8 Y_FORCE_INLINE get_8(TZCMemoryInput& in) { + static ui8 Y_FORCE_INLINE get_8(TZCMemoryInput& in) { return get<ui8>(in); } @@ -73,25 +73,25 @@ void Y_FORCE_INLINE PackUI32(IOutputStream& out, ui32 v) { out.Write(buf, size); } -template <class TStream> -struct TInputStream2Traits { +template <class TStream> +struct TInputStream2Traits { typedef Stream_traits TTraits; }; -template <> -struct TInputStream2Traits<TZCMemoryInput> { +template <> +struct TInputStream2Traits<TZCMemoryInput> { typedef TZCMemoryInput_traits TTraits; }; -template <class TStream> -void Y_FORCE_INLINE UnPackUI32(TStream& in, ui32& v) { +template <class TStream> +void Y_FORCE_INLINE UnPackUI32(TStream& in, ui32& v) { size_t size; UNPACK_28(v, in, TInputStream2Traits<TStream>::TTraits, size); - (void)size; + (void)size; } -template <class TStream> -ui32 Y_FORCE_INLINE UnPackUI32(TStream& in) { +template <class TStream> +ui32 Y_FORCE_INLINE UnPackUI32(TStream& in) { ui32 res; UnPackUI32(in, res); return res; diff --git a/library/cpp/packedtypes/packed_ut.cpp b/library/cpp/packedtypes/packed_ut.cpp index 70a22cf9c3..4f7bd4a9ca 100644 --- a/library/cpp/packedtypes/packed_ut.cpp +++ b/library/cpp/packedtypes/packed_ut.cpp @@ -1,15 +1,15 @@ -#include "packed.h" - +#include "packed.h" + #include <library/cpp/testing/unittest/registar.h> #include <util/system/defaults.h> #include <util/generic/ylimits.h> #include <util/generic/buffer.h> #include <util/stream/mem.h> -#include <util/stream/buffer.h> +#include <util/stream/buffer.h> namespace NPrivate { -#if 0 +#if 0 static ui64 gSeed = 42; template<typename T> @@ -21,8 +21,8 @@ static T PseudoRandom(T max = Max<T>()) { gSeed = gSeed ^ (ui64(max) << 17u); return gSeed % max; }; -#endif -} +#endif +} Y_UNIT_TEST_SUITE(TPackedTest) { void TestPackUi32Sub(ui32 v, const TVector<char>& p) { @@ -54,38 +54,38 @@ Y_UNIT_TEST_SUITE(TPackedTest) { v = 0; pv.resize(1); pv[0] = 0x0; - TestPackUi32Sub(v, pv); + TestPackUi32Sub(v, pv); v = 0x1600; pv.resize(2); pv[0] = 0x96; pv[1] = 0x00; - TestPackUi32Sub(v, pv); + TestPackUi32Sub(v, pv); v = 0xEF98; pv.resize(3); pv[0] = 0xC0; -#if defined(_big_endian_) +#if defined(_big_endian_) pv[1] = 0xEF; pv[2] = 0x98; #elif defined(_little_endian_) pv[1] = 0x98; pv[2] = 0xEF; #endif - TestPackUi32Sub(v, pv); + TestPackUi32Sub(v, pv); v = 0xF567FE4; pv.resize(4); pv[0] = 0xEF; pv[1] = 0x56; -#if defined(_big_endian_) +#if defined(_big_endian_) pv[2] = 0x7F; pv[3] = 0xE4; #elif defined(_little_endian_) pv[2] = 0xE4; pv[3] = 0x7F; #endif - TestPackUi32Sub(v, pv); + TestPackUi32Sub(v, pv); } #if 0 diff --git a/library/cpp/packedtypes/packedfloat.cpp b/library/cpp/packedtypes/packedfloat.cpp index 6039d78969..3f6d7379a4 100644 --- a/library/cpp/packedtypes/packedfloat.cpp +++ b/library/cpp/packedtypes/packedfloat.cpp @@ -2,10 +2,10 @@ #include <util/stream/output.h> -#define OUT_IMPL(T) \ - template <> \ - void Out<T>(IOutputStream & os, TTypeTraits<T>::TFuncParam val) { \ - os << (float)val; \ +#define OUT_IMPL(T) \ + template <> \ + void Out<T>(IOutputStream & os, TTypeTraits<T>::TFuncParam val) { \ + os << (float)val; \ } OUT_IMPL(f16) @@ -14,5 +14,5 @@ OUT_IMPL(f8) OUT_IMPL(uf8) OUT_IMPL(f8d) OUT_IMPL(uf8d) - -#undef OUT_IMPL + +#undef OUT_IMPL diff --git a/library/cpp/packedtypes/packedfloat.h b/library/cpp/packedtypes/packedfloat.h index f178912ed3..2f66d11130 100644 --- a/library/cpp/packedtypes/packedfloat.h +++ b/library/cpp/packedtypes/packedfloat.h @@ -1,86 +1,86 @@ #pragma once #include <util/generic/cast.h> -#include <util/generic/ylimits.h> +#include <util/generic/ylimits.h> #include <util/system/hi_lo.h> - + #include <cmath> #include <cfloat> -#include <limits> +#include <limits> #include <algorithm> #include <cassert> namespace NPackedFloat { - /* + /* Exponent Mantissa zero Mantissa non-zero Equation 0x00 zero denormal (-1)^sign * 2^-126 * 0.mantissa 0x01–0xfe normalized value (-1)^sign * 2^(exponent - 127) * 1.mantissa 0xff infinity NaN * */ - //fast 16 bit floats by melkov - template <ui8 SIGNED> - struct float16 { - private: - typedef float16<SIGNED> self; + //fast 16 bit floats by melkov + template <ui8 SIGNED> + struct float16 { + private: + typedef float16<SIGNED> self; - public: - ui16 val; + public: + ui16 val; - explicit float16(ui16 v = 0) - : val(v) - { - } + explicit float16(ui16 v = 0) + : val(v) + { + } - self& operator=(float t) { - assert(SIGNED == 1 || SIGNED == 0 && t >= 0.); + self& operator=(float t) { + assert(SIGNED == 1 || SIGNED == 0 && t >= 0.); val = BitCast<ui32>(t) >> (15 + SIGNED); - return *this; - } + return *this; + } - operator float() const { + operator float() const { return BitCast<float>((ui32)val << (15 + SIGNED)); - } + } - static self New(float v) { - self f; - return f = v; - } + static self New(float v) { + self f; + return f = v; + } - static self denorm_min() { - return self(0x0001); - } + static self denorm_min() { + return self(0x0001); + } - static self min() { - return self(SIGNED ? 0x0080 : 0x0100); - } + static self min() { + return self(SIGNED ? 0x0080 : 0x0100); + } - static self max() { - return self(SIGNED ? 0x7f7f : 0xfeff); - } + static self max() { + return self(SIGNED ? 0x7f7f : 0xfeff); + } }; - //fast 8 bit floats - template <ui8 SIGNED, ui8 DENORM = 0> - struct float8 { - private: - typedef float8<SIGNED, DENORM> self; - enum { - FMinExp = SIGNED ? 0x7c : 0x78, - FMaxExp = SIGNED ? 0x83 : 0x87, - MaxExp = SIGNED ? 0x70 : 0xf0, - }; - - public: - ui8 val; - - explicit float8(ui8 v = 0) - : val(v) - { - } - - self& operator=(float t) { - assert(SIGNED == 1 || SIGNED == 0 && t >= 0.); + //fast 8 bit floats + template <ui8 SIGNED, ui8 DENORM = 0> + struct float8 { + private: + typedef float8<SIGNED, DENORM> self; + enum { + FMinExp = SIGNED ? 0x7c : 0x78, + FMaxExp = SIGNED ? 0x83 : 0x87, + MaxExp = SIGNED ? 0x70 : 0xf0, + }; + + public: + ui8 val; + + explicit float8(ui8 v = 0) + : val(v) + { + } + + self& operator=(float t) { + assert(SIGNED == 1 || SIGNED == 0 && t >= 0.); ui16 hi16 = Hi16(t); ui8 sign = SIGNED ? Hi8(hi16) & 0x80 : 0; @@ -88,68 +88,68 @@ namespace NPackedFloat { hi16 <<= 1; ui8 fexp = Hi8(hi16); - ui8 exp; + ui8 exp; ui8 frac = (Lo8(hi16) & 0xf0) >> 4; - - if (fexp <= FMinExp) { - exp = 0; - frac = DENORM ? ((ui8)(0x10 | frac) >> std::min<int>((FMinExp - fexp + 1), 8)) : 0; - } else if (fexp > FMaxExp) { - exp = MaxExp; - frac = 0x0f; - } else { - exp = (fexp - FMinExp) << 4; - } - - val = sign | exp | frac; - return *this; + + if (fexp <= FMinExp) { + exp = 0; + frac = DENORM ? ((ui8)(0x10 | frac) >> std::min<int>((FMinExp - fexp + 1), 8)) : 0; + } else if (fexp > FMaxExp) { + exp = MaxExp; + frac = 0x0f; + } else { + exp = (fexp - FMinExp) << 4; + } + + val = sign | exp | frac; + return *this; } - operator float() const { + operator float() const { ui32 v = 0; v |= SIGNED ? (val & 0x80) << 24 : 0; - ui8 frac = val & 0x0f; - ui8 exp = val & MaxExp; + ui8 frac = val & 0x0f; + ui8 exp = val & MaxExp; - if (exp) { + if (exp) { v |= ((exp >> 4) + FMinExp) << 23 | frac << 19; - } else if (DENORM && val & 0x0f) { - while (!(frac & 0x10)) { - frac <<= 1; - ++exp; - } + } else if (DENORM && val & 0x0f) { + while (!(frac & 0x10)) { + frac <<= 1; + ++exp; + } v |= (FMinExp - exp + 1) << 23 | (frac & 0x0f) << 19; - } else + } else v |= 0; return BitCast<float>(v); - } + } - static self New(float v) { - self f; - return f = v; - } + static self New(float v) { + self f; + return f = v; + } - static self denorm_min() { - return self(0x01); - } + static self denorm_min() { + return self(0x01); + } - static self min() { - return self(0x10); - } + static self min() { + return self(0x10); + } - static self max() { - return self(SIGNED ? 0x7f : 0xff); - } - }; + static self max() { + return self(SIGNED ? 0x7f : 0xff); + } + }; } using f64 = double; using f32 = float; -static_assert(sizeof(f32) == 4, "expect sizeof(f32) == 4"); -static_assert(sizeof(f64) == 8, "expect sizeof(f64) == 8"); +static_assert(sizeof(f32) == 4, "expect sizeof(f32) == 4"); +static_assert(sizeof(f64) == 8, "expect sizeof(f64) == 8"); using f16 = NPackedFloat::float16<1>; using uf16 = NPackedFloat::float16<0>; using f8 = NPackedFloat::float8<1>; @@ -162,18 +162,18 @@ using frac8 = ui8; using frac16 = ui16; -template <class T> +template <class T> inline constexpr T Float2Frac(float fac) { - return T(fac * float(Max<T>())); + return T(fac * float(Max<T>())); } -template <class T> +template <class T> inline constexpr T Float2FracR(float fac) { - float v = fac * float(Max<T>()); + float v = fac * float(Max<T>()); return T(v + 0.5f); } -template <class T> +template <class T> inline constexpr float Frac2Float(T pf) { constexpr float multiplier = float(1.0 / Max<T>()); return pf * multiplier; @@ -206,12 +206,12 @@ inline float Frac2Float(ui8 pf) { template <> inline float Frac2Float(ui32 pf) = delete; -template <class T> -inline float FracOrFloatToFloat(T t) { +template <class T> +inline float FracOrFloatToFloat(T t) { return Frac2Float(t); } -template <> -inline float FracOrFloatToFloat<float>(float t) { +template <> +inline float FracOrFloatToFloat<float>(float t) { return t; } diff --git a/library/cpp/packedtypes/packedfloat_ut.cpp b/library/cpp/packedtypes/packedfloat_ut.cpp index f61e49014c..9f161c031c 100644 --- a/library/cpp/packedtypes/packedfloat_ut.cpp +++ b/library/cpp/packedtypes/packedfloat_ut.cpp @@ -1,21 +1,21 @@ -#include "packedfloat.h" - +#include "packedfloat.h" + #include <library/cpp/testing/unittest/registar.h> -#include <util/generic/ylimits.h> - +#include <util/generic/ylimits.h> + class TPackedFloatTest: public TTestBase { - UNIT_TEST_SUITE(TPackedFloatTest); - UNIT_TEST(F16Test); - UNIT_TEST(Uf16Test); - UNIT_TEST(F8dTest); - UNIT_TEST(F8Test); - UNIT_TEST(Uf8dTest); - UNIT_TEST(Uf8Test); - UNIT_TEST_SUITE_END(); - + UNIT_TEST_SUITE(TPackedFloatTest); + UNIT_TEST(F16Test); + UNIT_TEST(Uf16Test); + UNIT_TEST(F8dTest); + UNIT_TEST(F8Test); + UNIT_TEST(Uf8dTest); + UNIT_TEST(Uf8Test); + UNIT_TEST_SUITE_END(); + private: - template <typename F> + template <typename F> void TestF(float f, float rf) { UNIT_ASSERT(F::New(f) == rf); } @@ -34,7 +34,7 @@ private: TestF16(2.0f * f16::max(), std::numeric_limits<float>::infinity()); TestF16(0.5 * f16::min(), 0.5 * f16::min()); TestF16(0.5 * f16::denorm_min(), 0); - TestF16(-0.5 * f16::denorm_min(), 0); + TestF16(-0.5 * f16::denorm_min(), 0); TestF16(FLT_MIN, FLT_MIN); TestF16(FLT_MAX, f16::max()); TestF16(f16::min(), FLT_MIN); @@ -125,6 +125,6 @@ private: TestUf8(uf8::min(), uf8::min()); TestUf8(2.0 * uf8::max(), uf8::max()); TestUf8(0.5 * uf8::min(), 0); - } -}; -UNIT_TEST_SUITE_REGISTRATION(TPackedFloatTest); + } +}; +UNIT_TEST_SUITE_REGISTRATION(TPackedFloatTest); diff --git a/library/cpp/packedtypes/ya.make b/library/cpp/packedtypes/ya.make index 4c2c950619..b91af5624c 100644 --- a/library/cpp/packedtypes/ya.make +++ b/library/cpp/packedtypes/ya.make @@ -11,7 +11,7 @@ PEERDIR( SRCS( fixed_point.h - longs.cpp + longs.cpp packed.h packedfloat.cpp packedfloat.h diff --git a/library/cpp/packedtypes/zigzag.h b/library/cpp/packedtypes/zigzag.h index 548403f838..0f982199ad 100644 --- a/library/cpp/packedtypes/zigzag.h +++ b/library/cpp/packedtypes/zigzag.h @@ -2,8 +2,8 @@ #include <util/generic/typetraits.h> -#include <limits.h> - +#include <limits.h> + //! Convert signed values to unsigned. Convenient for further varint encoding. //! See https://developers.google.com/protocol-buffers/docs/encoding#types for details. diff --git a/library/cpp/packers/packers.cpp b/library/cpp/packers/packers.cpp index 54615a9e7f..ed8cbbb299 100644 --- a/library/cpp/packers/packers.cpp +++ b/library/cpp/packers/packers.cpp @@ -9,7 +9,7 @@ namespace NPackers { #define _X_64(X) _X_32(X), _X_32(X) #define _X_128(X) _X_64(X), _X_64(X) - const ui8 SkipTable[256] = {_X_128(1), _X_64(2), _X_32(3), _X_16(4), _X_8(5), _X_4(6), 7, 7, 8, 9}; + const ui8 SkipTable[256] = {_X_128(1), _X_64(2), _X_32(3), _X_16(4), _X_8(5), _X_4(6), 7, 7, 8, 9}; #undef _X_4 #undef _X_8 diff --git a/library/cpp/packers/packers.h b/library/cpp/packers/packers.h index 1bde1b59aa..50b32b7214 100644 --- a/library/cpp/packers/packers.h +++ b/library/cpp/packers/packers.h @@ -18,8 +18,8 @@ public: t = T(); } - void PackLeaf(char*, const T&, size_t) const { - } + void PackLeaf(char*, const T&, size_t) const { + } size_t MeasureLeaf(const T&) const { return 0; @@ -30,7 +30,7 @@ public: } }; -template <typename T> +template <typename T> class TAsIsPacker { // this packer is not really a packer... public: void UnpackLeaf(const char* p, T& t) const { @@ -44,7 +44,7 @@ public: Y_UNUSED(data); return sizeof(T); } - size_t SkipLeaf(const char*) const { + size_t SkipLeaf(const char*) const { return sizeof(T); } }; @@ -111,7 +111,7 @@ namespace NPackers { size_t len = 1; value >>= 7; - for (; value && len < MAX_SIZE; value >>= 7) + for (; value && len < MAX_SIZE; value >>= 7) ++len; return len; @@ -122,7 +122,7 @@ namespace NPackers { ui64 value = val; int lenmask = 0; - for (size_t i = len - 1; i; --i) { + for (size_t i = len - 1; i; --i) { buffer[i] = (char)(value & 0xFF); value >>= 8; lenmask = ((lenmask >> 1) | (1 << 7)); @@ -220,7 +220,7 @@ namespace NPackers { h.F = f; return ReverseBytes(h.U); } - + public: void UnpackLeaf(const char* c, TFloat& t) const { TUInt u = 0; @@ -242,10 +242,10 @@ namespace NPackers { }; } - class TFloatPacker: public NImpl::TFPPackerBase<float, ui32> { + class TFloatPacker: public NImpl::TFPPackerBase<float, ui32> { }; - class TDoublePacker: public NImpl::TFPPackerBase<double, ui64> { + class TDoublePacker: public NImpl::TFPPackerBase<double, ui64> { }; //------------------------------------------- @@ -260,7 +260,7 @@ namespace NPackers { size_t SkipLeaf(const char* p) const; }; - template <class TStringType> + template <class TStringType> inline void TStringPacker<TStringType>::UnpackLeaf(const char* buf, TStringType& t) const { size_t len; TIntegralPacker<size_t>().UnpackLeaf(buf, len); @@ -268,7 +268,7 @@ namespace NPackers { t = TStringType((const typename TStringType::char_type*)(buf + start), len); } - template <class TStringType> + template <class TStringType> inline void TStringPacker<TStringType>::PackLeaf(char* buf, const TStringType& str, size_t size) const { size_t len = str.size(); size_t lenChar = len * sizeof(typename TStringType::char_type); @@ -277,13 +277,13 @@ namespace NPackers { memcpy(buf + start, str.data(), lenChar); } - template <class TStringType> + template <class TStringType> inline size_t TStringPacker<TStringType>::MeasureLeaf(const TStringType& str) const { size_t len = str.size(); return TIntegralPacker<size_t>().MeasureLeaf(len) + len * sizeof(typename TStringType::char_type); } - template <class TStringType> + template <class TStringType> inline size_t TStringPacker<TStringType>::SkipLeaf(const char* buf) const { size_t result = TIntegralPacker<size_t>().SkipLeaf(buf); { @@ -294,7 +294,7 @@ namespace NPackers { return result; } - template <class T> + template <class T> class TPacker; // TContainerPacker --- for any container @@ -308,50 +308,50 @@ namespace NPackers { // Requirements to class EP: has methods as in any packer (UnpackLeaf, PackLeaf, MeasureLeaf, SkipLeaf) that // are applicable to C::value_type - template <typename T> + template <typename T> struct TContainerInfo { enum { IsVector = 0 }; }; - template <typename T> - struct TContainerInfo<std::vector<T>> { + template <typename T> + struct TContainerInfo<std::vector<T>> { enum { IsVector = 1 }; }; - template <typename T> - struct TContainerInfo<TVector<T>> { + template <typename T> + struct TContainerInfo<TVector<T>> { enum { IsVector = 1 }; }; - template <bool IsVector> + template <bool IsVector> class TContainerPackerHelper { }; - template <> + template <> class TContainerPackerHelper<false> { public: - template <class Packer, class Container> - static void UnpackLeaf(Packer& p, const char* buffer, Container& c) { - p.UnpackLeafSimple(buffer, c); + template <class Packer, class Container> + static void UnpackLeaf(Packer& p, const char* buffer, Container& c) { + p.UnpackLeafSimple(buffer, c); } }; - template <> + template <> class TContainerPackerHelper<true> { public: - template <class Packer, class Container> - static void UnpackLeaf(Packer& p, const char* buffer, Container& c) { - p.UnpackLeafVector(buffer, c); + template <class Packer, class Container> + static void UnpackLeaf(Packer& p, const char* buffer, Container& c) { + p.UnpackLeafVector(buffer, c); } }; - template <class C, class EP = TPacker<typename C::value_type>> + template <class C, class EP = TPacker<typename C::value_type>> class TContainerPacker { private: typedef C TContainer; @@ -362,7 +362,7 @@ namespace NPackers { void UnpackLeafVector(const char* buffer, TContainer& c) const; friend class TContainerPackerHelper<TContainerInfo<C>::IsVector>; - + public: void UnpackLeaf(const char* buffer, TContainer& c) const { TContainerPackerHelper<TContainerInfo<C>::IsVector>::UnpackLeaf(*this, buffer, c); @@ -372,8 +372,8 @@ namespace NPackers { size_t SkipLeaf(const char* buffer) const; }; - template <class C, class EP> - inline void TContainerPacker<C, EP>::UnpackLeafSimple(const char* buffer, C& result) const { + template <class C, class EP> + inline void TContainerPacker<C, EP>::UnpackLeafSimple(const char* buffer, C& result) const { size_t offset = TIntegralPacker<size_t>().SkipLeaf(buffer); // first value is the total size (not needed here) size_t len; TIntegralPacker<size_t>().UnpackLeaf(buffer + offset, len); @@ -389,8 +389,8 @@ namespace NPackers { } } - template <class C, class EP> - inline void TContainerPacker<C, EP>::UnpackLeafVector(const char* buffer, C& result) const { + template <class C, class EP> + inline void TContainerPacker<C, EP>::UnpackLeafVector(const char* buffer, C& result) const { size_t offset = TIntegralPacker<size_t>().SkipLeaf(buffer); // first value is the total size (not needed here) size_t len; TIntegralPacker<size_t>().UnpackLeaf(buffer + offset, len); @@ -403,7 +403,7 @@ namespace NPackers { } } - template <class C, class EP> + template <class C, class EP> inline void TContainerPacker<C, EP>::PackLeaf(char* buffer, const C& data, size_t size) const { size_t sizeOfSize = TIntegralPacker<size_t>().MeasureLeaf(size); TIntegralPacker<size_t>().PackLeaf(buffer, size, sizeOfSize); @@ -419,7 +419,7 @@ namespace NPackers { Y_ASSERT(curSize == size); } - template <class C, class EP> + template <class C, class EP> inline size_t TContainerPacker<C, EP>::MeasureLeaf(const C& data) const { size_t curSize = TIntegralPacker<size_t>().MeasureLeaf(data.size()); for (TElementIterator p = data.begin(); p != data.end(); p++) @@ -434,7 +434,7 @@ namespace NPackers { return curSize + extraSize; } - template <class C, class EP> + template <class C, class EP> inline size_t TContainerPacker<C, EP>::SkipLeaf(const char* buffer) const { size_t value; TIntegralPacker<size_t>().UnpackLeaf(buffer, value); @@ -444,11 +444,11 @@ namespace NPackers { // TPairPacker --- for std::pair<T1, T2> (any two types; can be nested) // TPacker<T1> and TPacker<T2> should be valid classes - template <class T1, class T2, class TPacker1 = TPacker<T1>, class TPacker2 = TPacker<T2>> + template <class T1, class T2, class TPacker1 = TPacker<T1>, class TPacker2 = TPacker<T2>> class TPairPacker { private: typedef std::pair<T1, T2> TMyPair; - + public: void UnpackLeaf(const char* buffer, TMyPair& pair) const; void PackLeaf(char* buffer, const TMyPair& data, size_t size) const; @@ -456,14 +456,14 @@ namespace NPackers { size_t SkipLeaf(const char* buffer) const; }; - template <class T1, class T2, class TPacker1, class TPacker2> + template <class T1, class T2, class TPacker1, class TPacker2> inline void TPairPacker<T1, T2, TPacker1, TPacker2>::UnpackLeaf(const char* buffer, std::pair<T1, T2>& pair) const { TPacker1().UnpackLeaf(buffer, pair.first); size_t size = TPacker1().SkipLeaf(buffer); TPacker2().UnpackLeaf(buffer + size, pair.second); } - template <class T1, class T2, class TPacker1, class TPacker2> + template <class T1, class T2, class TPacker1, class TPacker2> inline void TPairPacker<T1, T2, TPacker1, TPacker2>::PackLeaf(char* buffer, const std::pair<T1, T2>& data, size_t size) const { size_t size1 = TPacker1().MeasureLeaf(data.first); TPacker1().PackLeaf(buffer, data.first, size1); @@ -472,14 +472,14 @@ namespace NPackers { Y_ASSERT(size == size1 + size2); } - template <class T1, class T2, class TPacker1, class TPacker2> + template <class T1, class T2, class TPacker1, class TPacker2> inline size_t TPairPacker<T1, T2, TPacker1, TPacker2>::MeasureLeaf(const std::pair<T1, T2>& data) const { size_t size1 = TPacker1().MeasureLeaf(data.first); size_t size2 = TPacker2().MeasureLeaf(data.second); return size1 + size2; } - template <class T1, class T2, class TPacker1, class TPacker2> + template <class T1, class T2, class TPacker1, class TPacker2> inline size_t TPairPacker<T1, T2, TPacker1, TPacker2>::SkipLeaf(const char* buffer) const { size_t size1 = TPacker1().SkipLeaf(buffer); size_t size2 = TPacker2().SkipLeaf(buffer + size1); @@ -544,68 +544,68 @@ namespace NPackers { class TPackerImpl; template <class T> - class TPackerImpl<T, true>: public TIntegralPacker<T> { + class TPackerImpl<T, true>: public TIntegralPacker<T> { }; // No implementation for non-integral types. template <class T> - class TPacker: public TPackerImpl<T, std::is_integral<T>::value> { + class TPacker: public TPackerImpl<T, std::is_integral<T>::value> { }; - template <> - class TPacker<float>: public TAsIsPacker<float> { + template <> + class TPacker<float>: public TAsIsPacker<float> { }; - template <> - class TPacker<double>: public TAsIsPacker<double> { + template <> + class TPacker<double>: public TAsIsPacker<double> { }; template <> - class TPacker<TString>: public TStringPacker<TString> { + class TPacker<TString>: public TStringPacker<TString> { }; template <> - class TPacker<TUtf16String>: public TStringPacker<TUtf16String> { + class TPacker<TUtf16String>: public TStringPacker<TUtf16String> { }; template <> - class TPacker<TStringBuf>: public TStringPacker<TStringBuf> { + class TPacker<TStringBuf>: public TStringPacker<TStringBuf> { }; template <> - class TPacker<TWtringBuf>: public TStringPacker<TWtringBuf> { + class TPacker<TWtringBuf>: public TStringPacker<TWtringBuf> { }; template <class T> - class TPacker<std::vector<T>>: public TContainerPacker<std::vector<T>> { + class TPacker<std::vector<T>>: public TContainerPacker<std::vector<T>> { }; template <class T> - class TPacker<TVector<T>>: public TContainerPacker<TVector<T>> { + class TPacker<TVector<T>>: public TContainerPacker<TVector<T>> { }; template <class T> - class TPacker<std::list<T>>: public TContainerPacker<std::list<T>> { + class TPacker<std::list<T>>: public TContainerPacker<std::list<T>> { }; template <class T> - class TPacker<TList<T>>: public TContainerPacker<TList<T>> { + class TPacker<TList<T>>: public TContainerPacker<TList<T>> { }; template <class T> - class TPacker<std::set<T>>: public TContainerPacker<std::set<T>> { + class TPacker<std::set<T>>: public TContainerPacker<std::set<T>> { }; template <class T> - class TPacker<TSet<T>>: public TContainerPacker<TSet<T>> { + class TPacker<TSet<T>>: public TContainerPacker<TSet<T>> { }; template <class T1, class T2> - class TPacker<std::pair<T1, T2>>: public TPairPacker<T1, T2> { + class TPacker<std::pair<T1, T2>>: public TPairPacker<T1, T2> { }; template <class T, size_t N> - class TPacker<std::array<T, N>>: public TArrayPacker<std::array<T, N>> { + class TPacker<std::array<T, N>>: public TArrayPacker<std::array<T, N>> { }; -} +} diff --git a/library/cpp/packers/proto_packer.h b/library/cpp/packers/proto_packer.h index 5a3d008e29..799badda5b 100644 --- a/library/cpp/packers/proto_packer.h +++ b/library/cpp/packers/proto_packer.h @@ -47,4 +47,4 @@ namespace NPackers { return result; } }; -} +} diff --git a/library/cpp/packers/ut/packers_ut.cpp b/library/cpp/packers/ut/packers_ut.cpp index 18ce2150d1..65a817f9b2 100644 --- a/library/cpp/packers/ut/packers_ut.cpp +++ b/library/cpp/packers/ut/packers_ut.cpp @@ -1,11 +1,11 @@ #include <library/cpp/testing/unittest/registar.h> - + #include <util/stream/output.h> #include <utility> - -#include <util/charset/wide.h> + +#include <util/charset/wide.h> #include <util/generic/algorithm.h> -#include <util/generic/buffer.h> +#include <util/generic/buffer.h> #include <util/generic/map.h> #include <util/generic/vector.h> #include <util/generic/ptr.h> @@ -13,19 +13,19 @@ #include <util/folder/dirut.h> -#include <util/random/random.h> - +#include <util/random/random.h> + #include <util/string/hex.h> #include "packers.h" - + #include <array> #include <iterator> -class TPackersTest: public TTestBase { +class TPackersTest: public TTestBase { private: UNIT_TEST_SUITE(TPackersTest); - UNIT_TEST(TestPackers); + UNIT_TEST(TestPackers); UNIT_TEST_SUITE_END(); template <class TData, class TPacker> @@ -66,44 +66,44 @@ void TPackersTest::TestPacker(const TData* test, size_t size) { void TPackersTest::TestPackers() { { - const TString test[] = {"", - "a", "b", "c", "d", - "aa", "ab", "ac", "ad", - "aaa", "aab", "aac", "aad", - "aba", "abb", "abc", "abd", - "asdfjjmk.gjilsjgilsjilgjildsajgfilsjdfilgjm ldsa8oq43u 583uq4905 -q435 jiores u893q 5oiju fd-KE 89536 9Q2URE 12AI894T3 89 Q*(re43"}; + const TString test[] = {"", + "a", "b", "c", "d", + "aa", "ab", "ac", "ad", + "aaa", "aab", "aac", "aad", + "aba", "abb", "abc", "abd", + "asdfjjmk.gjilsjgilsjilgjildsajgfilsjdfilgjm ldsa8oq43u 583uq4905 -q435 jiores u893q 5oiju fd-KE 89536 9Q2URE 12AI894T3 89 Q*(re43"}; - TestPacker<TString, NPackers::TPacker<TString>>(test, Y_ARRAY_SIZE(test)); + TestPacker<TString, NPackers::TPacker<TString>>(test, Y_ARRAY_SIZE(test)); for (size_t i = 0; i != Y_ARRAY_SIZE(test); ++i) { - TestPacker<TUtf16String, NPackers::TPacker<TUtf16String>>(UTF8ToWide(test[i])); + TestPacker<TUtf16String, NPackers::TPacker<TUtf16String>>(UTF8ToWide(test[i])); } } { const ui64 test[] = { - 0, 1, 2, 3, 4, 5, 6, 76, 100000, Max<ui64>()}; + 0, 1, 2, 3, 4, 5, 6, 76, 100000, Max<ui64>()}; - TestPacker<ui64, NPackers::TPacker<ui64>>(test, Y_ARRAY_SIZE(test)); + TestPacker<ui64, NPackers::TPacker<ui64>>(test, Y_ARRAY_SIZE(test)); } { const int test[] = { - 0, 1, 2, 3, 4, 5, 6, 76, 100000, -1, -2, -3, -4, -5, -6, -76, -10000, Min<int>(), Max<int>()}; + 0, 1, 2, 3, 4, 5, 6, 76, 100000, -1, -2, -3, -4, -5, -6, -76, -10000, Min<int>(), Max<int>()}; - TestPacker<int, NPackers::TPacker<int>>(test, Y_ARRAY_SIZE(test)); + TestPacker<int, NPackers::TPacker<int>>(test, Y_ARRAY_SIZE(test)); } { const float test[] = { 2.f, 3.f, 4.f, 0.f, -0.f, 1.f, -1.f, 1.1f, -1.1f, - std::numeric_limits<float>::min(), -std::numeric_limits<float>::min(), - std::numeric_limits<float>::max(), -std::numeric_limits<float>::max()}; + std::numeric_limits<float>::min(), -std::numeric_limits<float>::min(), + std::numeric_limits<float>::max(), -std::numeric_limits<float>::max()}; TestPacker<float, NPackers::TFloatPacker>(test, Y_ARRAY_SIZE(test)); } { const double test[] = { 0., -0., 1., -1., 1.1, -1.1, - std::numeric_limits<double>::min(), -std::numeric_limits<double>::min(), - std::numeric_limits<double>::max(), -std::numeric_limits<double>::max()}; + std::numeric_limits<double>::min(), -std::numeric_limits<double>::min(), + std::numeric_limits<double>::max(), -std::numeric_limits<double>::max()}; TestPacker<double, NPackers::TDoublePacker>(test, Y_ARRAY_SIZE(test)); } diff --git a/library/cpp/packers/ut/proto_packer_ut.cpp b/library/cpp/packers/ut/proto_packer_ut.cpp index e4151ba68c..2c5838323b 100644 --- a/library/cpp/packers/ut/proto_packer_ut.cpp +++ b/library/cpp/packers/ut/proto_packer_ut.cpp @@ -26,7 +26,7 @@ void FillRepeatedFields(TTestMessage& msg) { } // do not want to use google/protobuf/util/message_differencer because of warnings -bool operator==(const TTestMessage& lhs, const TTestMessage& rhs) { +bool operator==(const TTestMessage& lhs, const TTestMessage& rhs) { if (lhs.GetRequiredString() != rhs.GetRequiredString() || lhs.GetRequiredInt32() != rhs.GetRequiredInt32() || lhs.HasOptionalString() != rhs.HasOptionalString() || diff --git a/library/cpp/pop_count/benchmark/main.cpp b/library/cpp/pop_count/benchmark/main.cpp index 41ea3c91cc..13720beb18 100644 --- a/library/cpp/pop_count/benchmark/main.cpp +++ b/library/cpp/pop_count/benchmark/main.cpp @@ -1,52 +1,52 @@ -#include <util/stream/output.h> -#include <util/datetime/cputimer.h> +#include <util/stream/output.h> +#include <util/datetime/cputimer.h> #include <util/system/type_name.h> - + #include <library/cpp/pop_count/popcount.h> #include <library/cpp/testing/benchmark/bench.h> - -template <class F, class I> -inline void DoRun(F&& f, I&& i) { - const ui64 n = i.Iterations(); - + +template <class F, class I> +inline void DoRun(F&& f, I&& i) { + const ui64 n = i.Iterations(); + for (ui64 j = 0; j < n; ++j) { Y_DO_NOT_OPTIMIZE_AWAY(f(j * (ui64)123456 + (ui64)1)); - } -} - -Y_CPU_BENCHMARK(PopCount_8, iface) { - DoRun([](ui8 x) { - return PopCount<ui8>(x); - }, - iface); -} - -Y_CPU_BENCHMARK(PopCount_16, iface) { - DoRun([](ui16 x) { - return PopCount<ui16>(x); - }, - iface); -} - -Y_CPU_BENCHMARK(PopCount_32, iface) { - DoRun([](ui32 x) { - return PopCount<ui32>(x); - }, - iface); -} - -Y_CPU_BENCHMARK(PopCount_64, iface) { - DoRun([](ui64 x) { - return PopCount<ui64>(x); - }, - iface); -} - -#if !defined(_MSC_VER) -Y_CPU_BENCHMARK(BUILTIN_64, iface) { - DoRun([](ui64 x) { - return __builtin_popcountll(x); - }, - iface); -} -#endif + } +} + +Y_CPU_BENCHMARK(PopCount_8, iface) { + DoRun([](ui8 x) { + return PopCount<ui8>(x); + }, + iface); +} + +Y_CPU_BENCHMARK(PopCount_16, iface) { + DoRun([](ui16 x) { + return PopCount<ui16>(x); + }, + iface); +} + +Y_CPU_BENCHMARK(PopCount_32, iface) { + DoRun([](ui32 x) { + return PopCount<ui32>(x); + }, + iface); +} + +Y_CPU_BENCHMARK(PopCount_64, iface) { + DoRun([](ui64 x) { + return PopCount<ui64>(x); + }, + iface); +} + +#if !defined(_MSC_VER) +Y_CPU_BENCHMARK(BUILTIN_64, iface) { + DoRun([](ui64 x) { + return __builtin_popcountll(x); + }, + iface); +} +#endif diff --git a/library/cpp/pop_count/benchmark/ya.make b/library/cpp/pop_count/benchmark/ya.make index 7fb54a519a..02b61bdeca 100644 --- a/library/cpp/pop_count/benchmark/ya.make +++ b/library/cpp/pop_count/benchmark/ya.make @@ -1,14 +1,14 @@ OWNER(g:util) Y_BENCHMARK() - -PEERDIR( - util/draft + +PEERDIR( + util/draft library/cpp/pop_count -) - -SRCS( - main.cpp -) - -END() +) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/pop_count/popcount.cpp b/library/cpp/pop_count/popcount.cpp index 49276424be..28743b4426 100644 --- a/library/cpp/pop_count/popcount.cpp +++ b/library/cpp/pop_count/popcount.cpp @@ -1,30 +1,30 @@ -#include "popcount.h" - +#include "popcount.h" + #include <util/system/defaults.h> -#include <util/system/yassert.h> - -#include <string.h> - -static const ui8 PopCountLUT8Impl[1 << 8] = { -#define B2(n) n, n + 1, n + 1, n + 2 -#define B4(n) B2(n), B2(n + 1), B2(n + 1), B2(n + 2) -#define B6(n) B4(n), B4(n + 1), B4(n + 1), B4(n + 2) - B6(0), B6(1), B6(1), B6(2)}; - -ui8 const* PopCountLUT8 = PopCountLUT8Impl; +#include <util/system/yassert.h> -#if !defined(_MSC_VER) -//ICE here for msvc +#include <string.h> + +static const ui8 PopCountLUT8Impl[1 << 8] = { +#define B2(n) n, n + 1, n + 1, n + 2 +#define B4(n) B2(n), B2(n + 1), B2(n + 1), B2(n + 2) +#define B6(n) B4(n), B4(n + 1), B4(n + 1), B4(n + 2) + B6(0), B6(1), B6(1), B6(2)}; -static const ui8 PopCountLUT16Impl[1 << 16] = { -#define B2(n) n, n + 1, n + 1, n + 2 -#define B4(n) B2(n), B2(n + 1), B2(n + 1), B2(n + 2) -#define B6(n) B4(n), B4(n + 1), B4(n + 1), B4(n + 2) -#define B8(n) B6(n), B6(n + 1), B6(n + 1), B6(n + 2) -#define B10(n) B8(n), B8(n + 1), B8(n + 1), B8(n + 2) -#define B12(n) B10(n), B10(n + 1), B10(n + 1), B10(n + 2) -#define B14(n) B12(n), B12(n + 1), B12(n + 1), B12(n + 2) - B14(0), B14(1), B14(1), B14(2)}; +ui8 const* PopCountLUT8 = PopCountLUT8Impl; + +#if !defined(_MSC_VER) +//ICE here for msvc + +static const ui8 PopCountLUT16Impl[1 << 16] = { +#define B2(n) n, n + 1, n + 1, n + 2 +#define B4(n) B2(n), B2(n + 1), B2(n + 1), B2(n + 2) +#define B6(n) B4(n), B4(n + 1), B4(n + 1), B4(n + 2) +#define B8(n) B6(n), B6(n + 1), B6(n + 1), B6(n + 2) +#define B10(n) B8(n), B8(n + 1), B8(n + 1), B8(n + 2) +#define B12(n) B10(n), B10(n + 1), B10(n + 1), B10(n + 2) +#define B14(n) B12(n), B12(n + 1), B12(n + 1), B12(n + 2) + B14(0), B14(1), B14(1), B14(2)}; -ui8 const* PopCountLUT16 = PopCountLUT16Impl; -#endif +ui8 const* PopCountLUT16 = PopCountLUT16Impl; +#endif diff --git a/library/cpp/pop_count/popcount.h b/library/cpp/pop_count/popcount.h index 3d67737ed2..9833f039ff 100644 --- a/library/cpp/pop_count/popcount.h +++ b/library/cpp/pop_count/popcount.h @@ -6,11 +6,11 @@ #include <util/system/hi_lo.h> #include <util/system/platform.h> -#if defined(_MSC_VER) -#include <intrin.h> +#if defined(_MSC_VER) +#include <intrin.h> #endif -static inline ui32 PopCountImpl(ui8 n) { +static inline ui32 PopCountImpl(ui8 n) { #if defined(_ppc64_) ui32 r; __asm__("popcntb %0, %1" @@ -19,35 +19,35 @@ static inline ui32 PopCountImpl(ui8 n) { :); return r; #else - extern ui8 const* PopCountLUT8; - return PopCountLUT8[n]; + extern ui8 const* PopCountLUT8; + return PopCountLUT8[n]; #endif -} +} -static inline ui32 PopCountImpl(ui16 n) { -#if defined(_MSC_VER) - return __popcnt16(n); +static inline ui32 PopCountImpl(ui16 n) { +#if defined(_MSC_VER) + return __popcnt16(n); #else - extern ui8 const* PopCountLUT16; - return PopCountLUT16[n]; + extern ui8 const* PopCountLUT16; + return PopCountLUT16[n]; #endif -} +} -static inline ui32 PopCountImpl(ui32 n) { -#if defined(_MSC_VER) - return __popcnt(n); +static inline ui32 PopCountImpl(ui32 n) { +#if defined(_MSC_VER) + return __popcnt(n); #else -#if defined(_x86_64_) +#if defined(_x86_64_) if (NX86::CachedHavePOPCNT()) { - ui32 r; - - __asm__("popcnt %1, %0;" - : "=r"(r) - : "r"(n) - :); - - return r; - } + ui32 r; + + __asm__("popcnt %1, %0;" + : "=r"(r) + : "r"(n) + :); + + return r; + } #else #if defined(_ppc64_) ui32 r; @@ -58,28 +58,28 @@ static inline ui32 PopCountImpl(ui32 n) { :); return r; +#endif #endif -#endif - + return PopCountImpl((ui16)Lo16(n)) + PopCountImpl((ui16)Hi16(n)); #endif -} +} -static inline ui32 PopCountImpl(ui64 n) { +static inline ui32 PopCountImpl(ui64 n) { #if defined(_MSC_VER) && !defined(_i386_) - return __popcnt64(n); + return __popcnt64(n); #else -#if defined(_x86_64_) +#if defined(_x86_64_) if (NX86::CachedHavePOPCNT()) { - ui64 r; - - __asm__("popcnt %1, %0;" - : "=r"(r) - : "r"(n) - :); - - return r; - } + ui64 r; + + __asm__("popcnt %1, %0;" + : "=r"(r) + : "r"(n) + :); + + return r; + } #else #if defined(_ppc64_) ui32 r; @@ -90,16 +90,16 @@ static inline ui32 PopCountImpl(ui64 n) { :); return r; +#endif #endif -#endif - + return PopCountImpl((ui32)Lo32(n)) + PopCountImpl((ui32)Hi32(n)); #endif -} +} -template <class T> -static inline ui32 PopCount(T n) { +template <class T> +static inline ui32 PopCount(T n) { using TCvt = TFixedWidthUnsignedInt<T>; - return PopCountImpl((TCvt)n); + return PopCountImpl((TCvt)n); } diff --git a/library/cpp/pop_count/popcount_ut.cpp b/library/cpp/pop_count/popcount_ut.cpp index 5cd6605411..df2d246de2 100644 --- a/library/cpp/pop_count/popcount_ut.cpp +++ b/library/cpp/pop_count/popcount_ut.cpp @@ -1,70 +1,70 @@ -#include "popcount.h" - +#include "popcount.h" + #include <library/cpp/testing/unittest/registar.h> - -#include <util/random/random.h> - + +#include <util/random/random.h> + Y_UNIT_TEST_SUITE(TestPopCount) { - template <class T> - static inline ui32 SlowPopCount(T t) { - ui32 ret = 0; - - while (t) { - if (t & T(1)) { - ++ret; - } - - t = t >> 1; - } - - return ret; - } - - template <class T> - static inline void Test() { - for (size_t i = 0; i < 10000; ++i) { - const T rndv = RandomNumber<T>(); - - UNIT_ASSERT_VALUES_EQUAL(SlowPopCount(rndv), PopCount(rndv)); - } - } - + template <class T> + static inline ui32 SlowPopCount(T t) { + ui32 ret = 0; + + while (t) { + if (t & T(1)) { + ++ret; + } + + t = t >> 1; + } + + return ret; + } + + template <class T> + static inline void Test() { + for (size_t i = 0; i < 10000; ++i) { + const T rndv = RandomNumber<T>(); + + UNIT_ASSERT_VALUES_EQUAL(SlowPopCount(rndv), PopCount(rndv)); + } + } + Y_UNIT_TEST(Test8) { - Test<ui8>(); - } - + Test<ui8>(); + } + Y_UNIT_TEST(Test16) { - Test<ui16>(); - } - + Test<ui16>(); + } + Y_UNIT_TEST(Test32) { - Test<ui32>(); - } - + Test<ui32>(); + } + Y_UNIT_TEST(Test64) { - Test<ui64>(); - } - + Test<ui64>(); + } + Y_UNIT_TEST(TestPopCount) { - UNIT_ASSERT_VALUES_EQUAL(PopCount(0), 0); - UNIT_ASSERT_VALUES_EQUAL(PopCount(1), 1); - UNIT_ASSERT_VALUES_EQUAL(PopCount(1 << 10), 1); - UNIT_ASSERT_VALUES_EQUAL(PopCount((1 << 10) + 1), 2); - UNIT_ASSERT_VALUES_EQUAL(PopCount(0xFFFF), 16); - UNIT_ASSERT_VALUES_EQUAL(PopCount(0xFFFFFFFF), 32); - UNIT_ASSERT_VALUES_EQUAL(PopCount(0x55555555), 16); - - UNIT_ASSERT_VALUES_EQUAL(0, PopCount(0ULL)); - UNIT_ASSERT_VALUES_EQUAL(1, PopCount(1ULL)); - UNIT_ASSERT_VALUES_EQUAL(16, PopCount(0xAAAAAAAAULL)); - UNIT_ASSERT_VALUES_EQUAL(32, PopCount(0xFFFFFFFFULL)); - UNIT_ASSERT_VALUES_EQUAL(32, PopCount(0xAAAAAAAAAAAAAAAAULL)); - UNIT_ASSERT_VALUES_EQUAL(64, PopCount(0xFFFFFFFFFFFFFFFFULL)); - - ui64 v = 0; - - for (int i = 0; i < 64; v |= 1ULL << i, ++i) { - UNIT_ASSERT_VALUES_EQUAL(i, PopCount(v)); - } - } -} + UNIT_ASSERT_VALUES_EQUAL(PopCount(0), 0); + UNIT_ASSERT_VALUES_EQUAL(PopCount(1), 1); + UNIT_ASSERT_VALUES_EQUAL(PopCount(1 << 10), 1); + UNIT_ASSERT_VALUES_EQUAL(PopCount((1 << 10) + 1), 2); + UNIT_ASSERT_VALUES_EQUAL(PopCount(0xFFFF), 16); + UNIT_ASSERT_VALUES_EQUAL(PopCount(0xFFFFFFFF), 32); + UNIT_ASSERT_VALUES_EQUAL(PopCount(0x55555555), 16); + + UNIT_ASSERT_VALUES_EQUAL(0, PopCount(0ULL)); + UNIT_ASSERT_VALUES_EQUAL(1, PopCount(1ULL)); + UNIT_ASSERT_VALUES_EQUAL(16, PopCount(0xAAAAAAAAULL)); + UNIT_ASSERT_VALUES_EQUAL(32, PopCount(0xFFFFFFFFULL)); + UNIT_ASSERT_VALUES_EQUAL(32, PopCount(0xAAAAAAAAAAAAAAAAULL)); + UNIT_ASSERT_VALUES_EQUAL(64, PopCount(0xFFFFFFFFFFFFFFFFULL)); + + ui64 v = 0; + + for (int i = 0; i < 64; v |= 1ULL << i, ++i) { + UNIT_ASSERT_VALUES_EQUAL(i, PopCount(v)); + } + } +} diff --git a/library/cpp/pop_count/ut/ya.make b/library/cpp/pop_count/ut/ya.make index f0e6c014e5..a2be7f7951 100644 --- a/library/cpp/pop_count/ut/ya.make +++ b/library/cpp/pop_count/ut/ya.make @@ -1,9 +1,9 @@ UNITTEST_FOR(library/cpp/pop_count) - + OWNER(g:util) - -SRCS( - popcount_ut.cpp -) - -END() + +SRCS( + popcount_ut.cpp +) + +END() diff --git a/library/cpp/pop_count/ya.make b/library/cpp/pop_count/ya.make index 0dec238979..6e86560c85 100644 --- a/library/cpp/pop_count/ya.make +++ b/library/cpp/pop_count/ya.make @@ -1,9 +1,9 @@ -LIBRARY() - +LIBRARY() + OWNER(g:util) - -SRCS( - popcount.cpp -) - -END() + +SRCS( + popcount.cpp +) + +END() diff --git a/library/cpp/protobuf/json/config.h b/library/cpp/protobuf/json/config.h index dc84fb4d5d..3bf0874360 100644 --- a/library/cpp/protobuf/json/config.h +++ b/library/cpp/protobuf/json/config.h @@ -9,16 +9,16 @@ #include <functional> namespace NProtobufJson { - struct TProto2JsonConfig { - using TSelf = TProto2JsonConfig; + struct TProto2JsonConfig { + using TSelf = TProto2JsonConfig; - bool FormatOutput = false; + bool FormatOutput = false; - enum MissingKeyMode { + enum MissingKeyMode { // Skip missing keys - MissingKeySkip = 0, + MissingKeySkip = 0, // Fill missing keys with json null value. - MissingKeyNull, + MissingKeyNull, // Use default value in any case. // If default value is not explicitly defined, use default type value: // i.e. 0 for integers, "" for strings @@ -29,31 +29,31 @@ namespace NProtobufJson { // Throw exception if required field is empty. // For repeated keys, same as MissingKeySkip MissingKeyExplicitDefaultThrowRequired - }; - MissingKeyMode MissingSingleKeyMode = MissingKeySkip; - MissingKeyMode MissingRepeatedKeyMode = MissingKeySkip; - - /// Add null value for missing fields (false by default). - bool AddMissingFields = false; - - enum EnumValueMode { - EnumNumber = 0, // default - EnumName, - EnumFullName, - EnumNameLowerCase, - EnumFullNameLowerCase, - }; - EnumValueMode EnumMode = EnumNumber; - - enum FldNameMode { - FieldNameOriginalCase = 0, // default - FieldNameLowerCase, - FieldNameUpperCase, - FieldNameCamelCase, + }; + MissingKeyMode MissingSingleKeyMode = MissingKeySkip; + MissingKeyMode MissingRepeatedKeyMode = MissingKeySkip; + + /// Add null value for missing fields (false by default). + bool AddMissingFields = false; + + enum EnumValueMode { + EnumNumber = 0, // default + EnumName, + EnumFullName, + EnumNameLowerCase, + EnumFullNameLowerCase, + }; + EnumValueMode EnumMode = EnumNumber; + + enum FldNameMode { + FieldNameOriginalCase = 0, // default + FieldNameLowerCase, + FieldNameUpperCase, + FieldNameCamelCase, FieldNameSnakeCase, // ABC -> a_b_c, UserID -> user_i_d FieldNameSnakeCaseDense // ABC -> abc, UserID -> user_id - }; - FldNameMode FieldNameMode = FieldNameOriginalCase; + }; + FldNameMode FieldNameMode = FieldNameOriginalCase; enum ExtFldNameMode { ExtFldNameFull = 0, // default, field.full_name() @@ -61,104 +61,104 @@ namespace NProtobufJson { }; ExtFldNameMode ExtensionFieldNameMode = ExtFldNameFull; - /// Use 'json_name' protobuf option for field name, mutually exclusive - /// with FieldNameMode. - bool UseJsonName = false; + /// Use 'json_name' protobuf option for field name, mutually exclusive + /// with FieldNameMode. + bool UseJsonName = false; - /// Transforms will be applied only to string values (== protobuf fields of string / bytes type). - /// yajl_encode_string will be used if no transforms are specified. - TVector<TStringTransformPtr> StringTransforms; + /// Transforms will be applied only to string values (== protobuf fields of string / bytes type). + /// yajl_encode_string will be used if no transforms are specified. + TVector<TStringTransformPtr> StringTransforms; - /// Print map as object, otherwise print it as array of key/value objects - bool MapAsObject = false; + /// Print map as object, otherwise print it as array of key/value objects + bool MapAsObject = false; /// Stringify long integers which are not exactly representable by float or double values - enum EStringifyLongNumbersMode { - StringifyLongNumbersNever = 0, // default - StringifyLongNumbersForFloat, - StringifyLongNumbersForDouble, - }; - EStringifyLongNumbersMode StringifyLongNumbers = StringifyLongNumbersNever; + enum EStringifyLongNumbersMode { + StringifyLongNumbersNever = 0, // default + StringifyLongNumbersForFloat, + StringifyLongNumbersForDouble, + }; + EStringifyLongNumbersMode StringifyLongNumbers = StringifyLongNumbersNever; - /// Custom field names generator. - TNameGenerator NameGenerator = {}; + /// Custom field names generator. + TNameGenerator NameGenerator = {}; - /// Custom enum values generator. - TEnumValueGenerator EnumValueGenerator = {}; + /// Custom enum values generator. + TEnumValueGenerator EnumValueGenerator = {}; bool WriteNanAsString = false; - TSelf& SetFormatOutput(bool format) { - FormatOutput = format; - return *this; - } - - TSelf& SetMissingSingleKeyMode(MissingKeyMode mode) { - MissingSingleKeyMode = mode; - return *this; - } - - TSelf& SetMissingRepeatedKeyMode(MissingKeyMode mode) { - MissingRepeatedKeyMode = mode; - return *this; - } - - TSelf& SetAddMissingFields(bool add) { - AddMissingFields = add; - return *this; - } - - TSelf& SetEnumMode(EnumValueMode mode) { - EnumMode = mode; - return *this; - } - - TSelf& SetFieldNameMode(FldNameMode mode) { - Y_ENSURE(mode == FieldNameOriginalCase || !UseJsonName, "FieldNameMode and UseJsonName are mutually exclusive"); - FieldNameMode = mode; - return *this; - } - - TSelf& SetUseJsonName(bool jsonName) { - Y_ENSURE(!jsonName || FieldNameMode == FieldNameOriginalCase, "FieldNameMode and UseJsonName are mutually exclusive"); - UseJsonName = jsonName; - return *this; - } + TSelf& SetFormatOutput(bool format) { + FormatOutput = format; + return *this; + } + + TSelf& SetMissingSingleKeyMode(MissingKeyMode mode) { + MissingSingleKeyMode = mode; + return *this; + } + + TSelf& SetMissingRepeatedKeyMode(MissingKeyMode mode) { + MissingRepeatedKeyMode = mode; + return *this; + } + + TSelf& SetAddMissingFields(bool add) { + AddMissingFields = add; + return *this; + } + + TSelf& SetEnumMode(EnumValueMode mode) { + EnumMode = mode; + return *this; + } + + TSelf& SetFieldNameMode(FldNameMode mode) { + Y_ENSURE(mode == FieldNameOriginalCase || !UseJsonName, "FieldNameMode and UseJsonName are mutually exclusive"); + FieldNameMode = mode; + return *this; + } + + TSelf& SetUseJsonName(bool jsonName) { + Y_ENSURE(!jsonName || FieldNameMode == FieldNameOriginalCase, "FieldNameMode and UseJsonName are mutually exclusive"); + UseJsonName = jsonName; + return *this; + } TSelf& SetExtensionFieldNameMode(ExtFldNameMode mode) { ExtensionFieldNameMode = mode; return *this; } - TSelf& AddStringTransform(TStringTransformPtr transform) { - StringTransforms.push_back(transform); - return *this; - } + TSelf& AddStringTransform(TStringTransformPtr transform) { + StringTransforms.push_back(transform); + return *this; + } - TSelf& SetMapAsObject(bool value) { - MapAsObject = value; - return *this; - } + TSelf& SetMapAsObject(bool value) { + MapAsObject = value; + return *this; + } - TSelf& SetStringifyLongNumbers(EStringifyLongNumbersMode stringify) { - StringifyLongNumbers = stringify; - return *this; - } + TSelf& SetStringifyLongNumbers(EStringifyLongNumbersMode stringify) { + StringifyLongNumbers = stringify; + return *this; + } - TSelf& SetNameGenerator(TNameGenerator callback) { - NameGenerator = callback; - return *this; - } + TSelf& SetNameGenerator(TNameGenerator callback) { + NameGenerator = callback; + return *this; + } - TSelf& SetEnumValueGenerator(TEnumValueGenerator callback) { - EnumValueGenerator = callback; - return *this; - } + TSelf& SetEnumValueGenerator(TEnumValueGenerator callback) { + EnumValueGenerator = callback; + return *this; + } TSelf& SetWriteNanAsString(bool value) { WriteNanAsString = value; return *this; } - }; + }; -} +} diff --git a/library/cpp/protobuf/json/field_option.h b/library/cpp/protobuf/json/field_option.h index c8a8bfbff5..6241452b6a 100644 --- a/library/cpp/protobuf/json/field_option.h +++ b/library/cpp/protobuf/json/field_option.h @@ -5,36 +5,36 @@ #include <google/protobuf/message.h> namespace NProtobufJson { - // Functor that defines whether given field has some option set to true - // - // Example: - // message T { - // optional stroka some_field = 1 [(some_option) = true]; - // } - // - template <typename TFieldOptionExtensionId> - class TFieldOptionFunctor { - public: - TFieldOptionFunctor(const TFieldOptionExtensionId& option, bool positive = true) - : Option(option) - , Positive(positive) - { - } + // Functor that defines whether given field has some option set to true + // + // Example: + // message T { + // optional stroka some_field = 1 [(some_option) = true]; + // } + // + template <typename TFieldOptionExtensionId> + class TFieldOptionFunctor { + public: + TFieldOptionFunctor(const TFieldOptionExtensionId& option, bool positive = true) + : Option(option) + , Positive(positive) + { + } - bool operator()(const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor* field) const { - const NProtoBuf::FieldOptions& opt = field->options(); - const bool val = opt.GetExtension(Option); - return Positive ? val : !val; - } + bool operator()(const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor* field) const { + const NProtoBuf::FieldOptions& opt = field->options(); + const bool val = opt.GetExtension(Option); + return Positive ? val : !val; + } - private: - const TFieldOptionExtensionId& Option; - bool Positive; - }; - - template <typename TFieldOptionExtensionId> - TFieldOptionFunctor<TFieldOptionExtensionId> MakeFieldOptionFunctor(const TFieldOptionExtensionId& option, bool positive = true) { - return TFieldOptionFunctor<TFieldOptionExtensionId>(option, positive); + private: + const TFieldOptionExtensionId& Option; + bool Positive; + }; + + template <typename TFieldOptionExtensionId> + TFieldOptionFunctor<TFieldOptionExtensionId> MakeFieldOptionFunctor(const TFieldOptionExtensionId& option, bool positive = true) { + return TFieldOptionFunctor<TFieldOptionExtensionId>(option, positive); } } diff --git a/library/cpp/protobuf/json/filter.h b/library/cpp/protobuf/json/filter.h index 9a3ddb54fe..eced62c01a 100644 --- a/library/cpp/protobuf/json/filter.h +++ b/library/cpp/protobuf/json/filter.h @@ -10,39 +10,39 @@ #include <functional> namespace NProtobufJson { - template <typename TBasePrinter = TProto2JsonPrinter> // TBasePrinter is assumed to be a TProto2JsonPrinter descendant - class TFilteringPrinter: public TBasePrinter { - public: - using TFieldPredicate = std::function<bool(const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor*)>; - - template <typename... TArgs> - TFilteringPrinter(TFieldPredicate isPrinted, TArgs&&... args) - : TBasePrinter(std::forward<TArgs>(args)...) - , IsPrinted(std::move(isPrinted)) - { - } - - virtual void PrintField(const NProtoBuf::Message& proto, - const NProtoBuf::FieldDescriptor& field, - IJsonOutput& json, - TStringBuf key) override { - if (key || IsPrinted(proto, &field)) - TBasePrinter::PrintField(proto, field, json, key); - } - - private: - TFieldPredicate IsPrinted; - }; - - inline void PrintWithFilter(const NProtoBuf::Message& msg, TFilteringPrinter<>::TFieldPredicate filter, IJsonOutput& output, const TProto2JsonConfig& config = TProto2JsonConfig()) { - TFilteringPrinter<> printer(std::move(filter), config); - printer.Print(msg, output); + template <typename TBasePrinter = TProto2JsonPrinter> // TBasePrinter is assumed to be a TProto2JsonPrinter descendant + class TFilteringPrinter: public TBasePrinter { + public: + using TFieldPredicate = std::function<bool(const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor*)>; + + template <typename... TArgs> + TFilteringPrinter(TFieldPredicate isPrinted, TArgs&&... args) + : TBasePrinter(std::forward<TArgs>(args)...) + , IsPrinted(std::move(isPrinted)) + { + } + + virtual void PrintField(const NProtoBuf::Message& proto, + const NProtoBuf::FieldDescriptor& field, + IJsonOutput& json, + TStringBuf key) override { + if (key || IsPrinted(proto, &field)) + TBasePrinter::PrintField(proto, field, json, key); + } + + private: + TFieldPredicate IsPrinted; + }; + + inline void PrintWithFilter(const NProtoBuf::Message& msg, TFilteringPrinter<>::TFieldPredicate filter, IJsonOutput& output, const TProto2JsonConfig& config = TProto2JsonConfig()) { + TFilteringPrinter<> printer(std::move(filter), config); + printer.Print(msg, output); } - inline TString PrintWithFilter(const NProtoBuf::Message& msg, TFilteringPrinter<>::TFieldPredicate filter, const TProto2JsonConfig& config = TProto2JsonConfig()) { - TString ret; - PrintWithFilter(msg, std::move(filter), *CreateJsonMapOutput(ret, config), config); - return ret; + inline TString PrintWithFilter(const NProtoBuf::Message& msg, TFilteringPrinter<>::TFieldPredicate filter, const TProto2JsonConfig& config = TProto2JsonConfig()) { + TString ret; + PrintWithFilter(msg, std::move(filter), *CreateJsonMapOutput(ret, config), config); + return ret; } } diff --git a/library/cpp/protobuf/json/inline.h b/library/cpp/protobuf/json/inline.h index e2d7bb6ef0..e5b0980227 100644 --- a/library/cpp/protobuf/json/inline.h +++ b/library/cpp/protobuf/json/inline.h @@ -48,68 +48,68 @@ #include <functional> namespace NProtobufJson { - template <typename TBasePrinter = TProto2JsonPrinter> // TBasePrinter is assumed to be a TProto2JsonPrinter descendant - class TInliningPrinter: public TBasePrinter { - public: - using TFieldPredicate = std::function<bool(const NProtoBuf::Message&, - const NProtoBuf::FieldDescriptor*)>; - - template <typename... TArgs> - TInliningPrinter(TFieldPredicate isInlined, TArgs&&... args) - : TBasePrinter(std::forward<TArgs>(args)...) - , IsInlined(std::move(isInlined)) - { - } - - virtual void PrintField(const NProtoBuf::Message& proto, - const NProtoBuf::FieldDescriptor& field, - IJsonOutput& json, - TStringBuf key) override { - const NProtoBuf::TConstField f(proto, &field); - if (!key && IsInlined(proto, &field) && ShouldPrint(f)) { - key = this->MakeKey(field); - json.WriteKey(key); - if (!field.is_repeated()) { - json.WriteRawJson(f.Get<TString>()); - } else { - json.BeginList(); - for (size_t i = 0, sz = f.Size(); i < sz; ++i) - json.WriteRawJson(f.Get<TString>(i)); - json.EndList(); - } + template <typename TBasePrinter = TProto2JsonPrinter> // TBasePrinter is assumed to be a TProto2JsonPrinter descendant + class TInliningPrinter: public TBasePrinter { + public: + using TFieldPredicate = std::function<bool(const NProtoBuf::Message&, + const NProtoBuf::FieldDescriptor*)>; + + template <typename... TArgs> + TInliningPrinter(TFieldPredicate isInlined, TArgs&&... args) + : TBasePrinter(std::forward<TArgs>(args)...) + , IsInlined(std::move(isInlined)) + { + } + + virtual void PrintField(const NProtoBuf::Message& proto, + const NProtoBuf::FieldDescriptor& field, + IJsonOutput& json, + TStringBuf key) override { + const NProtoBuf::TConstField f(proto, &field); + if (!key && IsInlined(proto, &field) && ShouldPrint(f)) { + key = this->MakeKey(field); + json.WriteKey(key); + if (!field.is_repeated()) { + json.WriteRawJson(f.Get<TString>()); + } else { + json.BeginList(); + for (size_t i = 0, sz = f.Size(); i < sz; ++i) + json.WriteRawJson(f.Get<TString>(i)); + json.EndList(); + } } else { - TBasePrinter::PrintField(proto, field, json, key); + TBasePrinter::PrintField(proto, field, json, key); } + } + + private: + bool ShouldPrint(const NProtoBuf::TConstField& f) const { + if (!f.IsString()) + ythrow yexception() << "TInliningPrinter: json field " + << f.Field()->name() << " should be a string"; + + if (f.HasValue()) + return true; + + // we may want write default value for given field in case of its absence + const auto& cfg = this->GetConfig(); + return (f.Field()->is_repeated() ? cfg.MissingRepeatedKeyMode : cfg.MissingSingleKeyMode) == TProto2JsonConfig::MissingKeyDefault; } - private: - bool ShouldPrint(const NProtoBuf::TConstField& f) const { - if (!f.IsString()) - ythrow yexception() << "TInliningPrinter: json field " - << f.Field()->name() << " should be a string"; - - if (f.HasValue()) - return true; - - // we may want write default value for given field in case of its absence - const auto& cfg = this->GetConfig(); - return (f.Field()->is_repeated() ? cfg.MissingRepeatedKeyMode : cfg.MissingSingleKeyMode) == TProto2JsonConfig::MissingKeyDefault; - } - - private: - TFieldPredicate IsInlined; - }; + private: + TFieldPredicate IsInlined; + }; - inline void PrintInlined(const NProtoBuf::Message& msg, TInliningPrinter<>::TFieldPredicate isInlined, IJsonOutput& output, const TProto2JsonConfig& config = TProto2JsonConfig()) { - TInliningPrinter<> printer(std::move(isInlined), config); - printer.Print(msg, output); - } + inline void PrintInlined(const NProtoBuf::Message& msg, TInliningPrinter<>::TFieldPredicate isInlined, IJsonOutput& output, const TProto2JsonConfig& config = TProto2JsonConfig()) { + TInliningPrinter<> printer(std::move(isInlined), config); + printer.Print(msg, output); + } - inline TString PrintInlined(const NProtoBuf::Message& msg, TInliningPrinter<>::TFieldPredicate isInlined, const TProto2JsonConfig& config = TProto2JsonConfig()) { - TString ret; - PrintInlined(msg, std::move(isInlined), *CreateJsonMapOutput(ret, config), config); - return ret; + inline TString PrintInlined(const NProtoBuf::Message& msg, TInliningPrinter<>::TFieldPredicate isInlined, const TProto2JsonConfig& config = TProto2JsonConfig()) { + TString ret; + PrintInlined(msg, std::move(isInlined), *CreateJsonMapOutput(ret, config), config); + return ret; } } diff --git a/library/cpp/protobuf/json/json2proto.cpp b/library/cpp/protobuf/json/json2proto.cpp index 640c10f5a5..4eaa035ce4 100644 --- a/library/cpp/protobuf/json/json2proto.cpp +++ b/library/cpp/protobuf/json/json2proto.cpp @@ -35,7 +35,7 @@ } static TString GetFieldName(const google::protobuf::FieldDescriptor& field, - const NProtobufJson::TJson2ProtoConfig& config) { + const NProtobufJson::TJson2ProtoConfig& config) { if (config.NameGenerator) { return config.NameGenerator(field); } @@ -81,7 +81,7 @@ static void JsonString2Field(const NJson::TJsonValue& json, google::protobuf::Message& proto, const google::protobuf::FieldDescriptor& field, - const NProtobufJson::TJson2ProtoConfig& config) { + const NProtobufJson::TJson2ProtoConfig& config) { using namespace google::protobuf; const Reflection* reflection = proto.GetReflection(); @@ -172,7 +172,7 @@ Json2SingleField(const NJson::TJsonValue& json, google::protobuf::Message& proto, const google::protobuf::FieldDescriptor& field, const NProtobufJson::TJson2ProtoConfig& config, - bool isMapValue = false) { + bool isMapValue = false) { using namespace google::protobuf; const Reflection* reflection = proto.GetReflection(); @@ -181,7 +181,7 @@ Json2SingleField(const NJson::TJsonValue& json, TString name; if (!isMapValue) { name = GetFieldName(field, config); - if (!json.Has(name) || json[name].GetType() == NJson::JSON_UNDEFINED || json[name].GetType() == NJson::JSON_NULL) { + if (!json.Has(name) || json[name].GetType() == NJson::JSON_UNDEFINED || json[name].GetType() == NJson::JSON_NULL) { if (field.is_required() && !field.has_default_value() && !reflection->HasField(proto, &field) && config.CheckRequiredFields) { ythrow yexception() << "JSON has no field for required field " << name << "."; @@ -202,27 +202,27 @@ Json2SingleField(const NJson::TJsonValue& json, JSON_TO_FIELD(CPPTYPE_FLOAT, field.name(), fieldJson, IsDouble, SetFloat, GetDouble); JSON_TO_FIELD(CPPTYPE_BOOL, field.name(), fieldJson, IsBoolean, SetBool, GetBoolean); - case FieldDescriptor::CPPTYPE_STRING: { - JsonString2Field(fieldJson, proto, field, config); - break; - } + case FieldDescriptor::CPPTYPE_STRING: { + JsonString2Field(fieldJson, proto, field, config); + break; + } - case FieldDescriptor::CPPTYPE_ENUM: { + case FieldDescriptor::CPPTYPE_ENUM: { JsonEnum2Field(fieldJson, proto, field, config); - break; - } + break; + } - case FieldDescriptor::CPPTYPE_MESSAGE: { - Message* innerProto = reflection->MutableMessage(&proto, &field); - Y_ASSERT(!!innerProto); + case FieldDescriptor::CPPTYPE_MESSAGE: { + Message* innerProto = reflection->MutableMessage(&proto, &field); + Y_ASSERT(!!innerProto); NProtobufJson::MergeJson2Proto(fieldJson, *innerProto, config); - break; - } + break; + } - default: - ythrow yexception() << "Unknown protobuf field type: " - << static_cast<int>(field.cpp_type()) << "."; + default: + ythrow yexception() << "Unknown protobuf field type: " + << static_cast<int>(field.cpp_type()) << "."; } } @@ -315,7 +315,7 @@ static void Json2RepeatedField(const NJson::TJsonValue& json, google::protobuf::Message& proto, const google::protobuf::FieldDescriptor& field, - const NProtobufJson::TJson2ProtoConfig& config) { + const NProtobufJson::TJson2ProtoConfig& config) { using namespace google::protobuf; TString name = GetFieldName(field, config); @@ -323,7 +323,7 @@ Json2RepeatedField(const NJson::TJsonValue& json, return; const NJson::TJsonValue& fieldJson = json[name]; - if (fieldJson.GetType() == NJson::JSON_UNDEFINED || fieldJson.GetType() == NJson::JSON_NULL) + if (fieldJson.GetType() == NJson::JSON_UNDEFINED || fieldJson.GetType() == NJson::JSON_NULL) return; bool isMap = fieldJson.GetType() == NJson::JSON_MAP; @@ -373,49 +373,49 @@ Json2RepeatedField(const NJson::TJsonValue& json, namespace NProtobufJson { void MergeJson2Proto(const NJson::TJsonValue& json, google::protobuf::Message& proto, const TJson2ProtoConfig& config) { - if (json.IsNull()) { - return; - } + if (json.IsNull()) { + return; + } - Y_ENSURE(json.IsMap(), "expected json map"); + Y_ENSURE(json.IsMap(), "expected json map"); - const google::protobuf::Descriptor* descriptor = proto.GetDescriptor(); - Y_ASSERT(!!descriptor); + const google::protobuf::Descriptor* descriptor = proto.GetDescriptor(); + Y_ASSERT(!!descriptor); - for (int f = 0, endF = descriptor->field_count(); f < endF; ++f) { - const google::protobuf::FieldDescriptor* field = descriptor->field(f); - Y_ASSERT(!!field); + for (int f = 0, endF = descriptor->field_count(); f < endF; ++f) { + const google::protobuf::FieldDescriptor* field = descriptor->field(f); + Y_ASSERT(!!field); - if (field->is_repeated()) { - Json2RepeatedField(json, proto, *field, config); - } else { - Json2SingleField(json, proto, *field, config); - } + if (field->is_repeated()) { + Json2RepeatedField(json, proto, *field, config); + } else { + Json2SingleField(json, proto, *field, config); + } } - if (!config.AllowUnknownFields) { - THashMap<TString, bool> knownFields; - for (int f = 0, endF = descriptor->field_count(); f < endF; ++f) { - const google::protobuf::FieldDescriptor* field = descriptor->field(f); - knownFields[GetFieldName(*field, config)] = 1; - } - for (const auto& f : json.GetMap()) { + if (!config.AllowUnknownFields) { + THashMap<TString, bool> knownFields; + for (int f = 0, endF = descriptor->field_count(); f < endF; ++f) { + const google::protobuf::FieldDescriptor* field = descriptor->field(f); + knownFields[GetFieldName(*field, config)] = 1; + } + for (const auto& f : json.GetMap()) { Y_ENSURE(knownFields.contains(f.first), "unknown field " << f.first); - } + } } } void MergeJson2Proto(const TStringBuf& json, google::protobuf::Message& proto, const TJson2ProtoConfig& config) { - NJson::TJsonReaderConfig jsonCfg; - jsonCfg.DontValidateUtf8 = true; + NJson::TJsonReaderConfig jsonCfg; + jsonCfg.DontValidateUtf8 = true; jsonCfg.AllowComments = config.AllowComments; - NJson::TJsonValue jsonValue; - ReadJsonTree(json, &jsonCfg, &jsonValue, /* throwOnError = */ true); + NJson::TJsonValue jsonValue; + ReadJsonTree(json, &jsonCfg, &jsonValue, /* throwOnError = */ true); MergeJson2Proto(jsonValue, proto, config); - } - + } + void Json2Proto(const NJson::TJsonValue& json, google::protobuf::Message& proto, const TJson2ProtoConfig& config) { proto.Clear(); MergeJson2Proto(json, proto, config); diff --git a/library/cpp/protobuf/json/json2proto.h b/library/cpp/protobuf/json/json2proto.h index 4c33498dfa..44b05644c0 100644 --- a/library/cpp/protobuf/json/json2proto.h +++ b/library/cpp/protobuf/json/json2proto.h @@ -5,30 +5,30 @@ #include <library/cpp/json/json_reader.h> #include <library/cpp/json/json_value.h> - + #include <util/stream/input.h> -#include <util/stream/str.h> +#include <util/stream/str.h> #include <util/stream/mem.h> -namespace google { - namespace protobuf { - class Message; - } -} +namespace google { + namespace protobuf { + class Message; + } +} namespace NProtobufJson { - struct TJson2ProtoConfig { - using TSelf = TJson2ProtoConfig; + struct TJson2ProtoConfig { + using TSelf = TJson2ProtoConfig; using TValueVectorizer = std::function<NJson::TJsonValue::TArray(const NJson::TJsonValue& jsonValue)>; - enum FldNameMode { - FieldNameOriginalCase = 0, // default - FieldNameLowerCase, - FieldNameUpperCase, - FieldNameCamelCase, + enum FldNameMode { + FieldNameOriginalCase = 0, // default + FieldNameLowerCase, + FieldNameUpperCase, + FieldNameCamelCase, FieldNameSnakeCase, // ABC -> a_b_c, UserID -> user_i_d FieldNameSnakeCaseDense // ABC -> abc, UserID -> user_id - }; + }; enum EnumValueMode { EnumCaseSensetive = 0, // default @@ -36,37 +36,37 @@ namespace NProtobufJson { EnumSnakeCaseInsensitive }; - TSelf& SetFieldNameMode(FldNameMode mode) { - Y_ENSURE(mode == FieldNameOriginalCase || !UseJsonName, "FieldNameMode and UseJsonName are mutually exclusive"); - FieldNameMode = mode; - return *this; - } + TSelf& SetFieldNameMode(FldNameMode mode) { + Y_ENSURE(mode == FieldNameOriginalCase || !UseJsonName, "FieldNameMode and UseJsonName are mutually exclusive"); + FieldNameMode = mode; + return *this; + } - TSelf& SetUseJsonName(bool jsonName) { - Y_ENSURE(!jsonName || FieldNameMode == FieldNameOriginalCase, "FieldNameMode and UseJsonName are mutually exclusive"); - UseJsonName = jsonName; - return *this; - } + TSelf& SetUseJsonName(bool jsonName) { + Y_ENSURE(!jsonName || FieldNameMode == FieldNameOriginalCase, "FieldNameMode and UseJsonName are mutually exclusive"); + UseJsonName = jsonName; + return *this; + } - TSelf& AddStringTransform(TStringTransformPtr transform) { - StringTransforms.push_back(transform); - return *this; - } + TSelf& AddStringTransform(TStringTransformPtr transform) { + StringTransforms.push_back(transform); + return *this; + } - TSelf& SetCastFromString(bool cast) { - CastFromString = cast; - return *this; - } + TSelf& SetCastFromString(bool cast) { + CastFromString = cast; + return *this; + } TSelf& SetDoNotCastEmptyStrings(bool cast) { DoNotCastEmptyStrings = cast; return *this; } - TSelf& SetCastRobust(bool cast) { - CastRobust = cast; - return *this; - } + TSelf& SetCastRobust(bool cast) { + CastRobust = cast; + return *this; + } TSelf& SetMapAsObject(bool mapAsObject) { MapAsObject = mapAsObject; @@ -78,10 +78,10 @@ namespace NProtobufJson { return *this; } - TSelf& SetNameGenerator(TNameGenerator callback) { - NameGenerator = callback; - return *this; - } + TSelf& SetNameGenerator(TNameGenerator callback) { + NameGenerator = callback; + return *this; + } TSelf& SetEnumValueMode(EnumValueMode enumValueMode) { EnumValueMode = enumValueMode; @@ -104,34 +104,34 @@ namespace NProtobufJson { } FldNameMode FieldNameMode = FieldNameOriginalCase; - bool AllowUnknownFields = true; + bool AllowUnknownFields = true; - /// Use 'json_name' protobuf option for field name, mutually exclusive - /// with FieldNameMode. - bool UseJsonName = false; + /// Use 'json_name' protobuf option for field name, mutually exclusive + /// with FieldNameMode. + bool UseJsonName = false; - /// Transforms will be applied only to string values (== protobuf fields of string / bytes type). - TVector<TStringTransformPtr> StringTransforms; + /// Transforms will be applied only to string values (== protobuf fields of string / bytes type). + TVector<TStringTransformPtr> StringTransforms; - /// Cast string json values to protobuf field type - bool CastFromString = false; + /// Cast string json values to protobuf field type + bool CastFromString = false; /// Skip empty strings, instead casting from string into scalar types. /// I.e. empty string like default value for scalar types. bool DoNotCastEmptyStrings = false; - /// Cast all json values to protobuf field types - bool CastRobust = false; + /// Cast all json values to protobuf field types + bool CastRobust = false; - /// Consider map to be an object, otherwise consider it to be an array of key/value objects - bool MapAsObject = false; + /// Consider map to be an object, otherwise consider it to be an array of key/value objects + bool MapAsObject = false; - /// Throw exception if there is no required fields in json object. - bool CheckRequiredFields = true; + /// Throw exception if there is no required fields in json object. + bool CheckRequiredFields = true; /// Replace repeated fields content during merging bool ReplaceRepeatedFields = false; - /// Custom field names generator. - TNameGenerator NameGenerator = {}; + /// Custom field names generator. + TNameGenerator NameGenerator = {}; /// Enum value parsing mode. EnumValueMode EnumValueMode = EnumCaseSensetive; @@ -144,9 +144,9 @@ namespace NProtobufJson { /// Allow js-style comments (both // and /**/) bool AllowComments = false; - }; + }; - /// @throw yexception + /// @throw yexception void MergeJson2Proto(const NJson::TJsonValue& json, google::protobuf::Message& proto, const TJson2ProtoConfig& config = TJson2ProtoConfig()); @@ -161,62 +161,62 @@ namespace NProtobufJson { } /// @throw yexception - void Json2Proto(const NJson::TJsonValue& json, google::protobuf::Message& proto, - const TJson2ProtoConfig& config = TJson2ProtoConfig()); + void Json2Proto(const NJson::TJsonValue& json, google::protobuf::Message& proto, + const TJson2ProtoConfig& config = TJson2ProtoConfig()); - /// @throw yexception - void Json2Proto(const TStringBuf& json, google::protobuf::Message& proto, - const TJson2ProtoConfig& config = TJson2ProtoConfig()); + /// @throw yexception + void Json2Proto(const TStringBuf& json, google::protobuf::Message& proto, + const TJson2ProtoConfig& config = TJson2ProtoConfig()); - /// @throw yexception - inline void Json2Proto(const TString& json, google::protobuf::Message& proto, - const TJson2ProtoConfig& config = TJson2ProtoConfig()) { - Json2Proto(TStringBuf(json), proto, config); - } + /// @throw yexception + inline void Json2Proto(const TString& json, google::protobuf::Message& proto, + const TJson2ProtoConfig& config = TJson2ProtoConfig()) { + Json2Proto(TStringBuf(json), proto, config); + } - /// @throw yexception + /// @throw yexception inline void Json2Proto(IInputStream& in, google::protobuf::Message& proto, const TJson2ProtoConfig& config = TJson2ProtoConfig()) { Json2Proto(TStringBuf(in.ReadAll()), proto, config); } /// @throw yexception - template <typename T> - T Json2Proto(IInputStream& in, const NJson::TJsonReaderConfig& readerConfig, - const TJson2ProtoConfig& config = TJson2ProtoConfig()) { - NJson::TJsonValue jsonValue; - NJson::ReadJsonTree(&in, &readerConfig, &jsonValue, true); - T protoValue; - Json2Proto(jsonValue, protoValue, config); - return protoValue; - } - - /// @throw yexception - template <typename T> - T Json2Proto(IInputStream& in, const TJson2ProtoConfig& config = TJson2ProtoConfig()) { - NJson::TJsonReaderConfig readerConfig; - readerConfig.DontValidateUtf8 = true; - return Json2Proto<T>(in, readerConfig, config); - } - - /// @throw yexception - template <typename T> - T Json2Proto(const TString& value, const TJson2ProtoConfig& config = TJson2ProtoConfig()) { - TStringInput in(value); - return Json2Proto<T>(in, config); - } - - /// @throw yexception - template <typename T> - T Json2Proto(const TStringBuf& value, const TJson2ProtoConfig& config = TJson2ProtoConfig()) { - TMemoryInput in(value); - return Json2Proto<T>(in, config); - } - - /// @throw yexception - template <typename T> - T Json2Proto(const char* ptr, const TJson2ProtoConfig& config = TJson2ProtoConfig()) { - return Json2Proto<T>(TStringBuf(ptr), config); - } + template <typename T> + T Json2Proto(IInputStream& in, const NJson::TJsonReaderConfig& readerConfig, + const TJson2ProtoConfig& config = TJson2ProtoConfig()) { + NJson::TJsonValue jsonValue; + NJson::ReadJsonTree(&in, &readerConfig, &jsonValue, true); + T protoValue; + Json2Proto(jsonValue, protoValue, config); + return protoValue; + } + + /// @throw yexception + template <typename T> + T Json2Proto(IInputStream& in, const TJson2ProtoConfig& config = TJson2ProtoConfig()) { + NJson::TJsonReaderConfig readerConfig; + readerConfig.DontValidateUtf8 = true; + return Json2Proto<T>(in, readerConfig, config); + } + + /// @throw yexception + template <typename T> + T Json2Proto(const TString& value, const TJson2ProtoConfig& config = TJson2ProtoConfig()) { + TStringInput in(value); + return Json2Proto<T>(in, config); + } + + /// @throw yexception + template <typename T> + T Json2Proto(const TStringBuf& value, const TJson2ProtoConfig& config = TJson2ProtoConfig()) { + TMemoryInput in(value); + return Json2Proto<T>(in, config); + } + + /// @throw yexception + template <typename T> + T Json2Proto(const char* ptr, const TJson2ProtoConfig& config = TJson2ProtoConfig()) { + return Json2Proto<T>(TStringBuf(ptr), config); + } } diff --git a/library/cpp/protobuf/json/json_output.h b/library/cpp/protobuf/json/json_output.h index df143af57a..8007909fda 100644 --- a/library/cpp/protobuf/json/json_output.h +++ b/library/cpp/protobuf/json/json_output.h @@ -4,76 +4,76 @@ #include <util/generic/strbuf.h> namespace NProtobufJson { - class IJsonOutput { - public: - template <typename T> - IJsonOutput& Write(const T& t) { - DoWrite(t); - return *this; - } - IJsonOutput& WriteNull() { - DoWriteNull(); - return *this; - } + class IJsonOutput { + public: + template <typename T> + IJsonOutput& Write(const T& t) { + DoWrite(t); + return *this; + } + IJsonOutput& WriteNull() { + DoWriteNull(); + return *this; + } - IJsonOutput& BeginList() { - DoBeginList(); - return *this; - } - IJsonOutput& EndList() { - DoEndList(); - return *this; - } + IJsonOutput& BeginList() { + DoBeginList(); + return *this; + } + IJsonOutput& EndList() { + DoEndList(); + return *this; + } - IJsonOutput& BeginObject() { - DoBeginObject(); - return *this; - } - IJsonOutput& WriteKey(const TStringBuf& key) { - DoWriteKey(key); - return *this; - } - IJsonOutput& EndObject() { - DoEndObject(); - return *this; - } + IJsonOutput& BeginObject() { + DoBeginObject(); + return *this; + } + IJsonOutput& WriteKey(const TStringBuf& key) { + DoWriteKey(key); + return *this; + } + IJsonOutput& EndObject() { + DoEndObject(); + return *this; + } - IJsonOutput& WriteRawJson(const TStringBuf& str) { - DoWriteRawJson(str); - return *this; - } + IJsonOutput& WriteRawJson(const TStringBuf& str) { + DoWriteRawJson(str); + return *this; + } - virtual ~IJsonOutput() { - } + virtual ~IJsonOutput() { + } - protected: - virtual void DoWrite(const TStringBuf& s) = 0; - virtual void DoWrite(const TString& s) = 0; - virtual void DoWrite(int i) = 0; - void DoWrite(long i) { - DoWrite(static_cast<long long>(i)); - } - virtual void DoWrite(long long i) = 0; - virtual void DoWrite(unsigned int i) = 0; - void DoWrite(unsigned long i) { - DoWrite(static_cast<unsigned long long>(i)); - } - virtual void DoWrite(unsigned long long i) = 0; - virtual void DoWrite(float f) = 0; - virtual void DoWrite(double f) = 0; - virtual void DoWrite(bool b) = 0; - virtual void DoWriteNull() = 0; + protected: + virtual void DoWrite(const TStringBuf& s) = 0; + virtual void DoWrite(const TString& s) = 0; + virtual void DoWrite(int i) = 0; + void DoWrite(long i) { + DoWrite(static_cast<long long>(i)); + } + virtual void DoWrite(long long i) = 0; + virtual void DoWrite(unsigned int i) = 0; + void DoWrite(unsigned long i) { + DoWrite(static_cast<unsigned long long>(i)); + } + virtual void DoWrite(unsigned long long i) = 0; + virtual void DoWrite(float f) = 0; + virtual void DoWrite(double f) = 0; + virtual void DoWrite(bool b) = 0; + virtual void DoWriteNull() = 0; - virtual void DoBeginList() = 0; - virtual void DoEndList() = 0; + virtual void DoBeginList() = 0; + virtual void DoEndList() = 0; - virtual void DoBeginObject() = 0; - virtual void DoWriteKey(const TStringBuf& key) = 0; - virtual void DoEndObject() = 0; + virtual void DoBeginObject() = 0; + virtual void DoWriteKey(const TStringBuf& key) = 0; + virtual void DoEndObject() = 0; - virtual void DoWriteRawJson(const TStringBuf& str) = 0; - }; + virtual void DoWriteRawJson(const TStringBuf& str) = 0; + }; - using TJsonMapOutputPtr = THolder<IJsonOutput>; + using TJsonMapOutputPtr = THolder<IJsonOutput>; -} +} diff --git a/library/cpp/protobuf/json/json_output_create.cpp b/library/cpp/protobuf/json/json_output_create.cpp index 378e4ea65a..7c84c97e76 100644 --- a/library/cpp/protobuf/json/json_output_create.cpp +++ b/library/cpp/protobuf/json/json_output_create.cpp @@ -5,28 +5,28 @@ #include "json_value_output.h" namespace NProtobufJson { - TJsonMapOutputPtr CreateJsonMapOutput(IOutputStream& out, const NJson::TJsonWriterConfig& config) { + TJsonMapOutputPtr CreateJsonMapOutput(IOutputStream& out, const NJson::TJsonWriterConfig& config) { return MakeHolder<TJsonWriterOutput>(&out, config); - } + } - TJsonMapOutputPtr CreateJsonMapOutput(NJson::TJsonWriter& writer) { + TJsonMapOutputPtr CreateJsonMapOutput(NJson::TJsonWriter& writer) { return MakeHolder<TBaseJsonWriterOutput>(writer); - } + } - TJsonMapOutputPtr CreateJsonMapOutput(TString& str, const TProto2JsonConfig& config) { + TJsonMapOutputPtr CreateJsonMapOutput(TString& str, const TProto2JsonConfig& config) { return MakeHolder<TJsonStringWriterOutput>(&str, config); - } + } - TJsonMapOutputPtr CreateJsonMapOutput(TStringStream& out, const TProto2JsonConfig& config) { + TJsonMapOutputPtr CreateJsonMapOutput(TStringStream& out, const TProto2JsonConfig& config) { return MakeHolder<TJsonWriterOutput>(&out, config); - } + } - TJsonMapOutputPtr CreateJsonMapOutput(IOutputStream& out, const TProto2JsonConfig& config) { + TJsonMapOutputPtr CreateJsonMapOutput(IOutputStream& out, const TProto2JsonConfig& config) { return MakeHolder<TJsonWriterOutput>(&out, config); - } + } - TJsonMapOutputPtr CreateJsonMapOutput(NJson::TJsonValue& json) { + TJsonMapOutputPtr CreateJsonMapOutput(NJson::TJsonValue& json) { return MakeHolder<TJsonValueOutput>(json); - } + } } diff --git a/library/cpp/protobuf/json/json_output_create.h b/library/cpp/protobuf/json/json_output_create.h index ad3889f5e9..ab63a89a11 100644 --- a/library/cpp/protobuf/json/json_output_create.h +++ b/library/cpp/protobuf/json/json_output_create.h @@ -4,19 +4,19 @@ #include "json_output.h" namespace NJson { - class TJsonValue; - class TJsonWriter; - struct TJsonWriterConfig; -} + class TJsonValue; + class TJsonWriter; + struct TJsonWriterConfig; +} class IOutputStream; class TStringStream; namespace NProtobufJson { - TJsonMapOutputPtr CreateJsonMapOutput(IOutputStream& out, const NJson::TJsonWriterConfig& config); - TJsonMapOutputPtr CreateJsonMapOutput(NJson::TJsonWriter& writer); - TJsonMapOutputPtr CreateJsonMapOutput(IOutputStream& out, const TProto2JsonConfig& config = TProto2JsonConfig()); - TJsonMapOutputPtr CreateJsonMapOutput(TString& str, const TProto2JsonConfig& config = TProto2JsonConfig()); - TJsonMapOutputPtr CreateJsonMapOutput(NJson::TJsonValue& json); + TJsonMapOutputPtr CreateJsonMapOutput(IOutputStream& out, const NJson::TJsonWriterConfig& config); + TJsonMapOutputPtr CreateJsonMapOutput(NJson::TJsonWriter& writer); + TJsonMapOutputPtr CreateJsonMapOutput(IOutputStream& out, const TProto2JsonConfig& config = TProto2JsonConfig()); + TJsonMapOutputPtr CreateJsonMapOutput(TString& str, const TProto2JsonConfig& config = TProto2JsonConfig()); + TJsonMapOutputPtr CreateJsonMapOutput(NJson::TJsonValue& json); -} +} diff --git a/library/cpp/protobuf/json/json_value_output.cpp b/library/cpp/protobuf/json/json_value_output.cpp index d845cc1c74..3ac95354d6 100644 --- a/library/cpp/protobuf/json/json_value_output.cpp +++ b/library/cpp/protobuf/json/json_value_output.cpp @@ -3,104 +3,104 @@ #include <library/cpp/json/json_reader.h> namespace NProtobufJson { - template <typename T> - void TJsonValueOutput::WriteImpl(const T& t) { - Y_ASSERT(Context.top().Type == TContext::JSON_ARRAY || Context.top().Type == TContext::JSON_AFTER_KEY); - - if (Context.top().Type == TContext::JSON_AFTER_KEY) { - Context.top().Value = t; - Context.pop(); - } else { - Context.top().Value.AppendValue(t); - } - } - - void TJsonValueOutput::DoWrite(const TStringBuf& s) { - WriteImpl(s); - } - - void TJsonValueOutput::DoWrite(const TString& s) { - WriteImpl(s); - } - - void TJsonValueOutput::DoWrite(int i) { - WriteImpl(i); - } - - void TJsonValueOutput::DoWrite(unsigned int i) { - WriteImpl(i); - } - - void TJsonValueOutput::DoWrite(long long i) { - WriteImpl(i); - } - - void TJsonValueOutput::DoWrite(unsigned long long i) { - WriteImpl(i); - } - - void TJsonValueOutput::DoWrite(float f) { - WriteImpl(f); - } - - void TJsonValueOutput::DoWrite(double f) { - WriteImpl(f); - } - - void TJsonValueOutput::DoWrite(bool b) { - WriteImpl(b); - } - - void TJsonValueOutput::DoWriteNull() { - WriteImpl(NJson::JSON_NULL); - } - - void TJsonValueOutput::DoBeginList() { - Y_ASSERT(Context.top().Type == TContext::JSON_ARRAY || Context.top().Type == TContext::JSON_AFTER_KEY); - - if (Context.top().Type == TContext::JSON_AFTER_KEY) { - Context.top().Type = TContext::JSON_ARRAY; - Context.top().Value.SetType(NJson::JSON_ARRAY); - } else { - Context.emplace(TContext::JSON_ARRAY, Context.top().Value.AppendValue(NJson::JSON_ARRAY)); - } - } - - void TJsonValueOutput::DoEndList() { - Y_ASSERT(Context.top().Type == TContext::JSON_ARRAY); - Context.pop(); - } - - void TJsonValueOutput::DoBeginObject() { - Y_ASSERT(Context.top().Type == TContext::JSON_ARRAY || Context.top().Type == TContext::JSON_AFTER_KEY); - - if (Context.top().Type == TContext::JSON_AFTER_KEY) { - Context.top().Type = TContext::JSON_MAP; - Context.top().Value.SetType(NJson::JSON_MAP); - } else { - Context.emplace(TContext::JSON_MAP, Context.top().Value.AppendValue(NJson::JSON_MAP)); - } - } - - void TJsonValueOutput::DoWriteKey(const TStringBuf& key) { - Y_ASSERT(Context.top().Type == TContext::JSON_MAP); - Context.emplace(TContext::JSON_AFTER_KEY, Context.top().Value[key]); - } - - void TJsonValueOutput::DoEndObject() { - Y_ASSERT(Context.top().Type == TContext::JSON_MAP); - Context.pop(); - } - - void TJsonValueOutput::DoWriteRawJson(const TStringBuf& str) { - Y_ASSERT(Context.top().Type == TContext::JSON_ARRAY || Context.top().Type == TContext::JSON_AFTER_KEY); - - if (Context.top().Type == TContext::JSON_AFTER_KEY) { - NJson::ReadJsonTree(str, &Context.top().Value); - Context.pop(); - } else { - NJson::ReadJsonTree(str, &Context.top().Value.AppendValue(NJson::JSON_UNDEFINED)); - } - } + template <typename T> + void TJsonValueOutput::WriteImpl(const T& t) { + Y_ASSERT(Context.top().Type == TContext::JSON_ARRAY || Context.top().Type == TContext::JSON_AFTER_KEY); + + if (Context.top().Type == TContext::JSON_AFTER_KEY) { + Context.top().Value = t; + Context.pop(); + } else { + Context.top().Value.AppendValue(t); + } + } + + void TJsonValueOutput::DoWrite(const TStringBuf& s) { + WriteImpl(s); + } + + void TJsonValueOutput::DoWrite(const TString& s) { + WriteImpl(s); + } + + void TJsonValueOutput::DoWrite(int i) { + WriteImpl(i); + } + + void TJsonValueOutput::DoWrite(unsigned int i) { + WriteImpl(i); + } + + void TJsonValueOutput::DoWrite(long long i) { + WriteImpl(i); + } + + void TJsonValueOutput::DoWrite(unsigned long long i) { + WriteImpl(i); + } + + void TJsonValueOutput::DoWrite(float f) { + WriteImpl(f); + } + + void TJsonValueOutput::DoWrite(double f) { + WriteImpl(f); + } + + void TJsonValueOutput::DoWrite(bool b) { + WriteImpl(b); + } + + void TJsonValueOutput::DoWriteNull() { + WriteImpl(NJson::JSON_NULL); + } + + void TJsonValueOutput::DoBeginList() { + Y_ASSERT(Context.top().Type == TContext::JSON_ARRAY || Context.top().Type == TContext::JSON_AFTER_KEY); + + if (Context.top().Type == TContext::JSON_AFTER_KEY) { + Context.top().Type = TContext::JSON_ARRAY; + Context.top().Value.SetType(NJson::JSON_ARRAY); + } else { + Context.emplace(TContext::JSON_ARRAY, Context.top().Value.AppendValue(NJson::JSON_ARRAY)); + } + } + + void TJsonValueOutput::DoEndList() { + Y_ASSERT(Context.top().Type == TContext::JSON_ARRAY); + Context.pop(); + } + + void TJsonValueOutput::DoBeginObject() { + Y_ASSERT(Context.top().Type == TContext::JSON_ARRAY || Context.top().Type == TContext::JSON_AFTER_KEY); + + if (Context.top().Type == TContext::JSON_AFTER_KEY) { + Context.top().Type = TContext::JSON_MAP; + Context.top().Value.SetType(NJson::JSON_MAP); + } else { + Context.emplace(TContext::JSON_MAP, Context.top().Value.AppendValue(NJson::JSON_MAP)); + } + } + + void TJsonValueOutput::DoWriteKey(const TStringBuf& key) { + Y_ASSERT(Context.top().Type == TContext::JSON_MAP); + Context.emplace(TContext::JSON_AFTER_KEY, Context.top().Value[key]); + } + + void TJsonValueOutput::DoEndObject() { + Y_ASSERT(Context.top().Type == TContext::JSON_MAP); + Context.pop(); + } + + void TJsonValueOutput::DoWriteRawJson(const TStringBuf& str) { + Y_ASSERT(Context.top().Type == TContext::JSON_ARRAY || Context.top().Type == TContext::JSON_AFTER_KEY); + + if (Context.top().Type == TContext::JSON_AFTER_KEY) { + NJson::ReadJsonTree(str, &Context.top().Value); + Context.pop(); + } else { + NJson::ReadJsonTree(str, &Context.top().Value.AppendValue(NJson::JSON_UNDEFINED)); + } + } } diff --git a/library/cpp/protobuf/json/json_value_output.h b/library/cpp/protobuf/json/json_value_output.h index 3fc6ff2ab0..8d2367eac7 100644 --- a/library/cpp/protobuf/json/json_value_output.h +++ b/library/cpp/protobuf/json/json_value_output.h @@ -7,57 +7,57 @@ #include <util/generic/stack.h> namespace NProtobufJson { - class TJsonValueOutput: public IJsonOutput { - public: - TJsonValueOutput(NJson::TJsonValue& value) - : Root(value) - { - Context.emplace(TContext::JSON_AFTER_KEY, Root); - } - - void DoWrite(const TStringBuf& s) override; - void DoWrite(const TString& s) override; - void DoWrite(int i) override; - void DoWrite(unsigned int i) override; - void DoWrite(long long i) override; - void DoWrite(unsigned long long i) override; - void DoWrite(float f) override; - void DoWrite(double f) override; - void DoWrite(bool b) override; - void DoWriteNull() override; - - void DoBeginList() override; - void DoEndList() override; - - void DoBeginObject() override; - void DoWriteKey(const TStringBuf& key) override; - void DoEndObject() override; - - void DoWriteRawJson(const TStringBuf& str) override; - - private: - template <typename T> - void WriteImpl(const T& t); - - struct TContext { - enum EType { - JSON_MAP, - JSON_ARRAY, - JSON_AFTER_KEY, - }; - - TContext(EType type, NJson::TJsonValue& value) - : Type(type) - , Value(value) - { - } - - EType Type; - NJson::TJsonValue& Value; + class TJsonValueOutput: public IJsonOutput { + public: + TJsonValueOutput(NJson::TJsonValue& value) + : Root(value) + { + Context.emplace(TContext::JSON_AFTER_KEY, Root); + } + + void DoWrite(const TStringBuf& s) override; + void DoWrite(const TString& s) override; + void DoWrite(int i) override; + void DoWrite(unsigned int i) override; + void DoWrite(long long i) override; + void DoWrite(unsigned long long i) override; + void DoWrite(float f) override; + void DoWrite(double f) override; + void DoWrite(bool b) override; + void DoWriteNull() override; + + void DoBeginList() override; + void DoEndList() override; + + void DoBeginObject() override; + void DoWriteKey(const TStringBuf& key) override; + void DoEndObject() override; + + void DoWriteRawJson(const TStringBuf& str) override; + + private: + template <typename T> + void WriteImpl(const T& t); + + struct TContext { + enum EType { + JSON_MAP, + JSON_ARRAY, + JSON_AFTER_KEY, + }; + + TContext(EType type, NJson::TJsonValue& value) + : Type(type) + , Value(value) + { + } + + EType Type; + NJson::TJsonValue& Value; }; - NJson::TJsonValue& Root; - TStack<TContext, TVector<TContext>> Context; + NJson::TJsonValue& Root; + TStack<TContext, TVector<TContext>> Context; }; -} +} diff --git a/library/cpp/protobuf/json/json_writer_output.cpp b/library/cpp/protobuf/json/json_writer_output.cpp index 288f645bab..b4ef84850c 100644 --- a/library/cpp/protobuf/json/json_writer_output.cpp +++ b/library/cpp/protobuf/json/json_writer_output.cpp @@ -1,22 +1,22 @@ #include "json_writer_output.h" namespace NProtobufJson { - NJson::TJsonWriterConfig TJsonWriterOutput::CreateJsonWriterConfig(const TProto2JsonConfig& config) { - NJson::TJsonWriterConfig jsonConfig; - jsonConfig.FormatOutput = config.FormatOutput; - jsonConfig.SortKeys = false; - jsonConfig.ValidateUtf8 = false; - jsonConfig.DontEscapeStrings = false; + NJson::TJsonWriterConfig TJsonWriterOutput::CreateJsonWriterConfig(const TProto2JsonConfig& config) { + NJson::TJsonWriterConfig jsonConfig; + jsonConfig.FormatOutput = config.FormatOutput; + jsonConfig.SortKeys = false; + jsonConfig.ValidateUtf8 = false; + jsonConfig.DontEscapeStrings = false; jsonConfig.WriteNanAsString = config.WriteNanAsString; - for (size_t i = 0; i < config.StringTransforms.size(); ++i) { - Y_ASSERT(config.StringTransforms[i]); - if (config.StringTransforms[i]->GetType() == IStringTransform::EscapeTransform) { - jsonConfig.DontEscapeStrings = true; - break; - } + for (size_t i = 0; i < config.StringTransforms.size(); ++i) { + Y_ASSERT(config.StringTransforms[i]); + if (config.StringTransforms[i]->GetType() == IStringTransform::EscapeTransform) { + jsonConfig.DontEscapeStrings = true; + break; + } } - return jsonConfig; + return jsonConfig; } - + } diff --git a/library/cpp/protobuf/json/json_writer_output.h b/library/cpp/protobuf/json/json_writer_output.h index 3d8a2daa56..bec4233018 100644 --- a/library/cpp/protobuf/json/json_writer_output.h +++ b/library/cpp/protobuf/json/json_writer_output.h @@ -9,95 +9,95 @@ #include <util/generic/store_policy.h> namespace NProtobufJson { - class TBaseJsonWriterOutput: public IJsonOutput { - public: - TBaseJsonWriterOutput(NJson::TJsonWriter& writer) - : Writer(writer) - { - } + class TBaseJsonWriterOutput: public IJsonOutput { + public: + TBaseJsonWriterOutput(NJson::TJsonWriter& writer) + : Writer(writer) + { + } - private: - void DoWrite(int i) override { - Writer.Write(i); - } - void DoWrite(unsigned int i) override { - Writer.Write(i); - } - void DoWrite(long long i) override { - Writer.Write(i); - } - void DoWrite(unsigned long long i) override { - Writer.Write(i); - } - void DoWrite(float f) override { - Writer.Write(f); - } - void DoWrite(double f) override { - Writer.Write(f); - } - void DoWrite(bool b) override { - Writer.Write(b); - } - void DoWriteNull() override { - Writer.WriteNull(); - } - void DoWrite(const TStringBuf& s) override { - Writer.Write(s); - } - void DoWrite(const TString& s) override { - Writer.Write(s); - } + private: + void DoWrite(int i) override { + Writer.Write(i); + } + void DoWrite(unsigned int i) override { + Writer.Write(i); + } + void DoWrite(long long i) override { + Writer.Write(i); + } + void DoWrite(unsigned long long i) override { + Writer.Write(i); + } + void DoWrite(float f) override { + Writer.Write(f); + } + void DoWrite(double f) override { + Writer.Write(f); + } + void DoWrite(bool b) override { + Writer.Write(b); + } + void DoWriteNull() override { + Writer.WriteNull(); + } + void DoWrite(const TStringBuf& s) override { + Writer.Write(s); + } + void DoWrite(const TString& s) override { + Writer.Write(s); + } - void DoBeginList() override { - Writer.OpenArray(); - } - void DoEndList() override { - Writer.CloseArray(); - } + void DoBeginList() override { + Writer.OpenArray(); + } + void DoEndList() override { + Writer.CloseArray(); + } - void DoBeginObject() override { - Writer.OpenMap(); - } - void DoWriteKey(const TStringBuf& key) override { - Writer.Write(key); - } - void DoEndObject() override { - Writer.CloseMap(); - } + void DoBeginObject() override { + Writer.OpenMap(); + } + void DoWriteKey(const TStringBuf& key) override { + Writer.Write(key); + } + void DoEndObject() override { + Writer.CloseMap(); + } - void DoWriteRawJson(const TStringBuf& str) override { - Writer.UnsafeWrite(str); - } + void DoWriteRawJson(const TStringBuf& str) override { + Writer.UnsafeWrite(str); + } - NJson::TJsonWriter& Writer; - }; + NJson::TJsonWriter& Writer; + }; - class TJsonWriterOutput: public TEmbedPolicy<NJson::TJsonWriter>, public TBaseJsonWriterOutput { - public: - TJsonWriterOutput(IOutputStream* outputStream, const NJson::TJsonWriterConfig& cfg) - : TEmbedPolicy<NJson::TJsonWriter>(outputStream, cfg) - , TBaseJsonWriterOutput(*Ptr()) - { - } + class TJsonWriterOutput: public TEmbedPolicy<NJson::TJsonWriter>, public TBaseJsonWriterOutput { + public: + TJsonWriterOutput(IOutputStream* outputStream, const NJson::TJsonWriterConfig& cfg) + : TEmbedPolicy<NJson::TJsonWriter>(outputStream, cfg) + , TBaseJsonWriterOutput(*Ptr()) + { + } - TJsonWriterOutput(IOutputStream* outputStream, const TProto2JsonConfig& cfg) - : TEmbedPolicy<NJson::TJsonWriter>(outputStream, CreateJsonWriterConfig(cfg)) - , TBaseJsonWriterOutput(*Ptr()) - { - } + TJsonWriterOutput(IOutputStream* outputStream, const TProto2JsonConfig& cfg) + : TEmbedPolicy<NJson::TJsonWriter>(outputStream, CreateJsonWriterConfig(cfg)) + , TBaseJsonWriterOutput(*Ptr()) + { + } - private: - static NJson::TJsonWriterConfig CreateJsonWriterConfig(const TProto2JsonConfig& cfg); - }; + private: + static NJson::TJsonWriterConfig CreateJsonWriterConfig(const TProto2JsonConfig& cfg); + }; - class TJsonStringWriterOutput: public TEmbedPolicy<TStringOutput>, public TJsonWriterOutput { - public: - template <typename TConfig> - TJsonStringWriterOutput(TString* str, const TConfig& cfg) - : TEmbedPolicy<TStringOutput>(*str) - , TJsonWriterOutput(TEmbedPolicy<TStringOutput>::Ptr(), cfg) - { - } - }; + class TJsonStringWriterOutput: public TEmbedPolicy<TStringOutput>, public TJsonWriterOutput { + public: + template <typename TConfig> + TJsonStringWriterOutput(TString* str, const TConfig& cfg) + : TEmbedPolicy<TStringOutput>(*str) + , TJsonWriterOutput(TEmbedPolicy<TStringOutput>::Ptr(), cfg) + { + } + }; -} +} diff --git a/library/cpp/protobuf/json/name_generator.h b/library/cpp/protobuf/json/name_generator.h index 2b5361bee2..d238283d03 100644 --- a/library/cpp/protobuf/json/name_generator.h +++ b/library/cpp/protobuf/json/name_generator.h @@ -5,14 +5,14 @@ #include <functional> namespace google { - namespace protobuf { - class FieldDescriptor; - class EnumValueDescriptor; - } -} + namespace protobuf { + class FieldDescriptor; + class EnumValueDescriptor; + } +} namespace NProtobufJson { - using TNameGenerator = std::function<TString(const google::protobuf::FieldDescriptor&)>; - using TEnumValueGenerator = std::function<TString(const google::protobuf::EnumValueDescriptor&)>; + using TNameGenerator = std::function<TString(const google::protobuf::FieldDescriptor&)>; + using TEnumValueGenerator = std::function<TString(const google::protobuf::EnumValueDescriptor&)>; -} +} diff --git a/library/cpp/protobuf/json/proto2json.cpp b/library/cpp/protobuf/json/proto2json.cpp index 3d76a91686..3083bd3422 100644 --- a/library/cpp/protobuf/json/proto2json.cpp +++ b/library/cpp/protobuf/json/proto2json.cpp @@ -14,43 +14,43 @@ #include <util/system/yassert.h> namespace NProtobufJson { - void Proto2Json(const NProtoBuf::Message& proto, IJsonOutput& jsonOutput, - const TProto2JsonConfig& config, bool closeMap) { - TProto2JsonPrinter printer(config); - printer.Print(proto, jsonOutput, closeMap); - } - - void Proto2Json(const NProtoBuf::Message& proto, NJson::TJsonValue& json, - const TProto2JsonConfig& config) { - Proto2Json(proto, *CreateJsonMapOutput(json), config); - } - - void Proto2Json(const NProtoBuf::Message& proto, NJson::TJsonWriter& writer, - const TProto2JsonConfig& config) { - Proto2Json(proto, *CreateJsonMapOutput(writer), config); - writer.Flush(); - } - - void Proto2Json(const NProtoBuf::Message& proto, IOutputStream& out, - const TProto2JsonConfig& config) { - Proto2Json(proto, *CreateJsonMapOutput(out, config), config); - } - - void Proto2Json(const NProtoBuf::Message& proto, TStringStream& out, - const TProto2JsonConfig& config) { - Proto2Json(proto, *CreateJsonMapOutput(out, config), config); - } - - void Proto2Json(const NProtoBuf::Message& proto, TString& str, - const TProto2JsonConfig& config) { - Proto2Json(proto, *CreateJsonMapOutput(str, config), config); - } - - TString Proto2Json(const ::NProtoBuf::Message& proto, - const TProto2JsonConfig& config) { - TString res; - Proto2Json(proto, res, config); - return res; - } + void Proto2Json(const NProtoBuf::Message& proto, IJsonOutput& jsonOutput, + const TProto2JsonConfig& config, bool closeMap) { + TProto2JsonPrinter printer(config); + printer.Print(proto, jsonOutput, closeMap); + } + + void Proto2Json(const NProtoBuf::Message& proto, NJson::TJsonValue& json, + const TProto2JsonConfig& config) { + Proto2Json(proto, *CreateJsonMapOutput(json), config); + } + + void Proto2Json(const NProtoBuf::Message& proto, NJson::TJsonWriter& writer, + const TProto2JsonConfig& config) { + Proto2Json(proto, *CreateJsonMapOutput(writer), config); + writer.Flush(); + } + + void Proto2Json(const NProtoBuf::Message& proto, IOutputStream& out, + const TProto2JsonConfig& config) { + Proto2Json(proto, *CreateJsonMapOutput(out, config), config); + } + + void Proto2Json(const NProtoBuf::Message& proto, TStringStream& out, + const TProto2JsonConfig& config) { + Proto2Json(proto, *CreateJsonMapOutput(out, config), config); + } + + void Proto2Json(const NProtoBuf::Message& proto, TString& str, + const TProto2JsonConfig& config) { + Proto2Json(proto, *CreateJsonMapOutput(str, config), config); + } + + TString Proto2Json(const ::NProtoBuf::Message& proto, + const TProto2JsonConfig& config) { + TString res; + Proto2Json(proto, res, config); + return res; + } } diff --git a/library/cpp/protobuf/json/proto2json.h b/library/cpp/protobuf/json/proto2json.h index 89a1781a40..55ee4e0846 100644 --- a/library/cpp/protobuf/json/proto2json.h +++ b/library/cpp/protobuf/json/proto2json.h @@ -15,64 +15,64 @@ #include <functional> namespace NJson { - class TJsonValue; - class TJsonWriter; -} + class TJsonValue; + class TJsonWriter; +} class IOutputStream; class TStringStream; namespace NProtobufJson { - void Proto2Json(const NProtoBuf::Message& proto, IJsonOutput& jsonOutput, - const TProto2JsonConfig& config = TProto2JsonConfig(), bool closeMap = true); + void Proto2Json(const NProtoBuf::Message& proto, IJsonOutput& jsonOutput, + const TProto2JsonConfig& config = TProto2JsonConfig(), bool closeMap = true); - void Proto2Json(const NProtoBuf::Message& proto, NJson::TJsonWriter& writer, - const TProto2JsonConfig& config = TProto2JsonConfig()); + void Proto2Json(const NProtoBuf::Message& proto, NJson::TJsonWriter& writer, + const TProto2JsonConfig& config = TProto2JsonConfig()); - /// @throw yexception - void Proto2Json(const NProtoBuf::Message& proto, NJson::TJsonValue& json, - const TProto2JsonConfig& config = TProto2JsonConfig()); + /// @throw yexception + void Proto2Json(const NProtoBuf::Message& proto, NJson::TJsonValue& json, + const TProto2JsonConfig& config = TProto2JsonConfig()); - /// @throw yexception - void Proto2Json(const NProtoBuf::Message& proto, IOutputStream& out, - const TProto2JsonConfig& config); - // Generated code shortcut - template <class T> - inline void Proto2Json(const T& proto, IOutputStream& out) { - out << proto.AsJSON(); - } + /// @throw yexception + void Proto2Json(const NProtoBuf::Message& proto, IOutputStream& out, + const TProto2JsonConfig& config); + // Generated code shortcut + template <class T> + inline void Proto2Json(const T& proto, IOutputStream& out) { + out << proto.AsJSON(); + } - // TStringStream deserves a special overload as its operator TString() would cause ambiguity - /// @throw yexception - void Proto2Json(const NProtoBuf::Message& proto, TStringStream& out, - const TProto2JsonConfig& config); - // Generated code shortcut - template <class T> - inline void Proto2Json(const T& proto, TStringStream& out) { - out << proto.AsJSON(); - } + // TStringStream deserves a special overload as its operator TString() would cause ambiguity + /// @throw yexception + void Proto2Json(const NProtoBuf::Message& proto, TStringStream& out, + const TProto2JsonConfig& config); + // Generated code shortcut + template <class T> + inline void Proto2Json(const T& proto, TStringStream& out) { + out << proto.AsJSON(); + } - /// @throw yexception - void Proto2Json(const NProtoBuf::Message& proto, TString& str, - const TProto2JsonConfig& config); - // Generated code shortcut - template <class T> - inline void Proto2Json(const T& proto, TString& str) { - str.clear(); - TStringOutput out(str); - out << proto.AsJSON(); - } + /// @throw yexception + void Proto2Json(const NProtoBuf::Message& proto, TString& str, + const TProto2JsonConfig& config); + // Generated code shortcut + template <class T> + inline void Proto2Json(const T& proto, TString& str) { + str.clear(); + TStringOutput out(str); + out << proto.AsJSON(); + } - /// @throw yexception - TString Proto2Json(const NProtoBuf::Message& proto, - const TProto2JsonConfig& config); + /// @throw yexception + TString Proto2Json(const NProtoBuf::Message& proto, + const TProto2JsonConfig& config); // Returns incorrect result if proto contains another NProtoBuf::Message - // Generated code shortcut - template <class T> - inline TString Proto2Json(const T& proto) { - TString result; - Proto2Json(proto, result); - return result; - } + // Generated code shortcut + template <class T> + inline TString Proto2Json(const T& proto) { + TString result; + Proto2Json(proto, result); + return result; + } } diff --git a/library/cpp/protobuf/json/proto2json_printer.cpp b/library/cpp/protobuf/json/proto2json_printer.cpp index 6123eab0f2..c2e2f1814f 100644 --- a/library/cpp/protobuf/json/proto2json_printer.cpp +++ b/library/cpp/protobuf/json/proto2json_printer.cpp @@ -192,15 +192,15 @@ namespace NProtobufJson { break; \ } -#define INT_FIELD_TO_JSON(EProtoCppType, ProtoGet) \ - case FieldDescriptor::EProtoCppType: { \ - const auto value = reflection->ProtoGet(proto, &field); \ - if (NeedStringifyNumber(value)) { \ - json.WriteKey(key).Write(ToString(value)); \ - } else { \ - json.WriteKey(key).Write(value); \ - } \ - break; \ +#define INT_FIELD_TO_JSON(EProtoCppType, ProtoGet) \ + case FieldDescriptor::EProtoCppType: { \ + const auto value = reflection->ProtoGet(proto, &field); \ + if (NeedStringifyNumber(value)) { \ + json.WriteKey(key).Write(ToString(value)); \ + } else { \ + json.WriteKey(key).Write(value); \ + } \ + break; \ } const Reflection* reflection = proto.GetReflection(); diff --git a/library/cpp/protobuf/json/string_transform.cpp b/library/cpp/protobuf/json/string_transform.cpp index 7c42daa677..6f982ece61 100644 --- a/library/cpp/protobuf/json/string_transform.cpp +++ b/library/cpp/protobuf/json/string_transform.cpp @@ -5,53 +5,53 @@ #include <library/cpp/string_utils/base64/base64.h> namespace NProtobufJson { - void TCEscapeTransform::Transform(TString& str) const { - str = google::protobuf::CEscape(str); - } + void TCEscapeTransform::Transform(TString& str) const { + str = google::protobuf::CEscape(str); + } - void TSafeUtf8CEscapeTransform::Transform(TString& str) const { - str = google::protobuf::strings::Utf8SafeCEscape(str); - } + void TSafeUtf8CEscapeTransform::Transform(TString& str) const { + str = google::protobuf::strings::Utf8SafeCEscape(str); + } - void TDoubleEscapeTransform::Transform(TString& str) const { - TString escaped = google::protobuf::CEscape(str); - str = ""; - for (char* it = escaped.begin(); *it; ++it) { - if (*it == '\\' || *it == '\"') - str += "\\"; - str += *it; - } - } + void TDoubleEscapeTransform::Transform(TString& str) const { + TString escaped = google::protobuf::CEscape(str); + str = ""; + for (char* it = escaped.begin(); *it; ++it) { + if (*it == '\\' || *it == '\"') + str += "\\"; + str += *it; + } + } - void TDoubleUnescapeTransform::Transform(TString& str) const { - str = google::protobuf::UnescapeCEscapeString(Unescape(str)); + void TDoubleUnescapeTransform::Transform(TString& str) const { + str = google::protobuf::UnescapeCEscapeString(Unescape(str)); } - TString TDoubleUnescapeTransform::Unescape(const TString& str) const { - if (str.empty()) { - return str; - } + TString TDoubleUnescapeTransform::Unescape(const TString& str) const { + if (str.empty()) { + return str; + } - TString result; - result.reserve(str.size()); + TString result; + result.reserve(str.size()); - char prev = str[0]; - bool doneOutput = true; - for (const char* it = str.c_str() + 1; *it; ++it) { - if (doneOutput && prev == '\\' && (*it == '\\' || *it == '\"')) { - doneOutput = false; - } else { - result += prev; - doneOutput = true; - } - prev = *it; - } + char prev = str[0]; + bool doneOutput = true; + for (const char* it = str.c_str() + 1; *it; ++it) { + if (doneOutput && prev == '\\' && (*it == '\\' || *it == '\"')) { + doneOutput = false; + } else { + result += prev; + doneOutput = true; + } + prev = *it; + } - if ((doneOutput && prev != '\\') || !doneOutput) { + if ((doneOutput && prev != '\\') || !doneOutput) { result += prev; } - return result; + return result; } void TBase64EncodeBytesTransform::TransformBytes(TString &str) const { diff --git a/library/cpp/protobuf/json/string_transform.h b/library/cpp/protobuf/json/string_transform.h index e4b296bc01..76042ddc23 100644 --- a/library/cpp/protobuf/json/string_transform.h +++ b/library/cpp/protobuf/json/string_transform.h @@ -5,83 +5,83 @@ #include <util/generic/refcount.h> namespace NProtobufJson { - class IStringTransform: public TSimpleRefCount<IStringTransform> { - public: - virtual ~IStringTransform() { - } - - /// Some transforms have special meaning. - /// For example, escape transforms cause generic JSON escaping to be turned off. - enum Type { - EscapeTransform = 0x1, - }; - - virtual int GetType() const = 0; - - /// This method is called for each string field in proto - virtual void Transform(TString& str) const = 0; - - /// This method is called for each bytes field in proto - virtual void TransformBytes(TString& str) const { - // Default behaviour is to apply string transform - return Transform(str); - } - }; - - using TStringTransformPtr = TIntrusivePtr<IStringTransform>; - - template <bool quote, bool tounicode> - class TEscapeJTransform: public IStringTransform { - public: - int GetType() const override { - return EscapeTransform; - } - - void Transform(TString& str) const override { - TString newStr; - NEscJ::EscapeJ<quote, tounicode>(str, newStr); - str = newStr; - } - }; - - class TCEscapeTransform: public IStringTransform { - public: - int GetType() const override { - return EscapeTransform; - } - - void Transform(TString& str) const override; - }; - - class TSafeUtf8CEscapeTransform: public IStringTransform { - public: - int GetType() const override { - return EscapeTransform; - } - - void Transform(TString& str) const override; - }; - - class TDoubleEscapeTransform: public IStringTransform { - public: - int GetType() const override { - return EscapeTransform; - } - - void Transform(TString& str) const override; - }; - - class TDoubleUnescapeTransform: public NProtobufJson::IStringTransform { - public: - int GetType() const override { - return NProtobufJson::IStringTransform::EscapeTransform; - } - - void Transform(TString& str) const override; - - private: - TString Unescape(const TString& str) const; - }; + class IStringTransform: public TSimpleRefCount<IStringTransform> { + public: + virtual ~IStringTransform() { + } + + /// Some transforms have special meaning. + /// For example, escape transforms cause generic JSON escaping to be turned off. + enum Type { + EscapeTransform = 0x1, + }; + + virtual int GetType() const = 0; + + /// This method is called for each string field in proto + virtual void Transform(TString& str) const = 0; + + /// This method is called for each bytes field in proto + virtual void TransformBytes(TString& str) const { + // Default behaviour is to apply string transform + return Transform(str); + } + }; + + using TStringTransformPtr = TIntrusivePtr<IStringTransform>; + + template <bool quote, bool tounicode> + class TEscapeJTransform: public IStringTransform { + public: + int GetType() const override { + return EscapeTransform; + } + + void Transform(TString& str) const override { + TString newStr; + NEscJ::EscapeJ<quote, tounicode>(str, newStr); + str = newStr; + } + }; + + class TCEscapeTransform: public IStringTransform { + public: + int GetType() const override { + return EscapeTransform; + } + + void Transform(TString& str) const override; + }; + + class TSafeUtf8CEscapeTransform: public IStringTransform { + public: + int GetType() const override { + return EscapeTransform; + } + + void Transform(TString& str) const override; + }; + + class TDoubleEscapeTransform: public IStringTransform { + public: + int GetType() const override { + return EscapeTransform; + } + + void Transform(TString& str) const override; + }; + + class TDoubleUnescapeTransform: public NProtobufJson::IStringTransform { + public: + int GetType() const override { + return NProtobufJson::IStringTransform::EscapeTransform; + } + + void Transform(TString& str) const override; + + private: + TString Unescape(const TString& str) const; + }; class TBase64EncodeBytesTransform: public NProtobufJson::IStringTransform { public: @@ -108,4 +108,4 @@ namespace NProtobufJson { void TransformBytes(TString &str) const override; }; -} +} diff --git a/library/cpp/protobuf/json/ut/filter_ut.cpp b/library/cpp/protobuf/json/ut/filter_ut.cpp index 95c227666f..0e2c13649a 100644 --- a/library/cpp/protobuf/json/ut/filter_ut.cpp +++ b/library/cpp/protobuf/json/ut/filter_ut.cpp @@ -22,72 +22,72 @@ static NProtobufJsonUt::TFilterTest GetTestMsg() { Y_UNIT_TEST_SUITE(TProto2JsonFilterTest){ Y_UNIT_TEST(TestFilterPrinter){ NProtobufJsonUt::TFilterTest msg = GetTestMsg(); -{ - TString expected = R"({"OptFiltered":"1","NotFiltered":"23","RepFiltered":[45,67],)" - R"("Inner":{"Number":[100,200],"InnerFiltered":235}})"; - TString my = Proto2Json(msg); - UNIT_ASSERT_STRINGS_EQUAL(my, expected); -} - -{ - TString expected = R"({"NotFiltered":"23",)" - R"("Inner":{"Number":[100,200]}})"; - TString my = PrintWithFilter(msg, MakeFieldOptionFunctor(NProtobufJsonUt::filter_test, false)); - UNIT_ASSERT_STRINGS_EQUAL(my, expected); -} +{ + TString expected = R"({"OptFiltered":"1","NotFiltered":"23","RepFiltered":[45,67],)" + R"("Inner":{"Number":[100,200],"InnerFiltered":235}})"; + TString my = Proto2Json(msg); + UNIT_ASSERT_STRINGS_EQUAL(my, expected); +} -{ - TString expected = R"({"OptFiltered":"1","RepFiltered":[45,67]})"; - TString my = PrintWithFilter(msg, MakeFieldOptionFunctor(NProtobufJsonUt::filter_test)); - UNIT_ASSERT_STRINGS_EQUAL(my, expected); -} +{ + TString expected = R"({"NotFiltered":"23",)" + R"("Inner":{"Number":[100,200]}})"; + TString my = PrintWithFilter(msg, MakeFieldOptionFunctor(NProtobufJsonUt::filter_test, false)); + UNIT_ASSERT_STRINGS_EQUAL(my, expected); +} -{ - TString expected = R"({"OptFiltered":"1","NotFiltered":"23",)" - R"("Inner":{"Number":[100,200]}})"; - TString my; - PrintWithFilter(msg, MakeFieldOptionFunctor(NProtobufJsonUt::export_test), *CreateJsonMapOutput(my)); - UNIT_ASSERT_STRINGS_EQUAL(my, expected); -} - -{ - TString expected = R"({"NotFiltered":"23",)" - R"("Inner":{"Number":[100,200]}})"; - auto functor = [](const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor* field) { - return field->name() == "NotFiltered" || field->name() == "Number" || field->name() == "Inner"; - }; - TString my = PrintWithFilter(msg, functor); - UNIT_ASSERT_STRINGS_EQUAL(my, expected); -} -} +{ + TString expected = R"({"OptFiltered":"1","RepFiltered":[45,67]})"; + TString my = PrintWithFilter(msg, MakeFieldOptionFunctor(NProtobufJsonUt::filter_test)); + UNIT_ASSERT_STRINGS_EQUAL(my, expected); +} +{ + TString expected = R"({"OptFiltered":"1","NotFiltered":"23",)" + R"("Inner":{"Number":[100,200]}})"; + TString my; + PrintWithFilter(msg, MakeFieldOptionFunctor(NProtobufJsonUt::export_test), *CreateJsonMapOutput(my)); + UNIT_ASSERT_STRINGS_EQUAL(my, expected); +} + +{ + TString expected = R"({"NotFiltered":"23",)" + R"("Inner":{"Number":[100,200]}})"; + auto functor = [](const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor* field) { + return field->name() == "NotFiltered" || field->name() == "Number" || field->name() == "Inner"; + }; + TString my = PrintWithFilter(msg, functor); + UNIT_ASSERT_STRINGS_EQUAL(my, expected); +} +} + Y_UNIT_TEST(NoUnnecessaryCopyFunctor) { - size_t CopyCount = 0; - struct TFunctorMock { - TFunctorMock(size_t* copyCount) - : CopyCount(copyCount) + size_t CopyCount = 0; + struct TFunctorMock { + TFunctorMock(size_t* copyCount) + : CopyCount(copyCount) { - UNIT_ASSERT(*CopyCount <= 1); + UNIT_ASSERT(*CopyCount <= 1); } - TFunctorMock(const TFunctorMock& f) - : CopyCount(f.CopyCount) + TFunctorMock(const TFunctorMock& f) + : CopyCount(f.CopyCount) { - ++*CopyCount; + ++*CopyCount; } - TFunctorMock(TFunctorMock&& f) = default; + TFunctorMock(TFunctorMock&& f) = default; - bool operator()(const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor*) const { - return false; - } + bool operator()(const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor*) const { + return false; + } - size_t* CopyCount; - }; + size_t* CopyCount; + }; - TProto2JsonConfig cfg; - TFilteringPrinter<> printer(TFunctorMock(&CopyCount), cfg); - UNIT_ASSERT(CopyCount <= 1); -} -} -; + TProto2JsonConfig cfg; + TFilteringPrinter<> printer(TFunctorMock(&CopyCount), cfg); + UNIT_ASSERT(CopyCount <= 1); +} +} +; diff --git a/library/cpp/protobuf/json/ut/inline_ut.cpp b/library/cpp/protobuf/json/ut/inline_ut.cpp index c29ad32e7d..0960de3220 100644 --- a/library/cpp/protobuf/json/ut/inline_ut.cpp +++ b/library/cpp/protobuf/json/ut/inline_ut.cpp @@ -24,99 +24,99 @@ static NProtobufJsonUt::TInlineTest GetTestMsg() { Y_UNIT_TEST_SUITE(TProto2JsonInlineTest){ Y_UNIT_TEST(TestNormalPrint){ NProtobufJsonUt::TInlineTest msg = GetTestMsg(); -// normal print should output these fields as just string values -TString expRaw = R"({"OptJson":"{\"a\":1,\"b\":\"000\"}","NotJson":"12{}34","RepJson":["{}","[1,2]"],)" - R"("Inner":{"Number":[100,200],"InnerJson":"{\"xxx\":[]}"}})"; -TString myRaw; -Proto2Json(msg, myRaw); -UNIT_ASSERT_STRINGS_EQUAL(myRaw, expRaw); - -myRaw = PrintInlined(msg, [](const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor*) { return false; }); -UNIT_ASSERT_STRINGS_EQUAL(myRaw, expRaw); // result is the same -} +// normal print should output these fields as just string values +TString expRaw = R"({"OptJson":"{\"a\":1,\"b\":\"000\"}","NotJson":"12{}34","RepJson":["{}","[1,2]"],)" + R"("Inner":{"Number":[100,200],"InnerJson":"{\"xxx\":[]}"}})"; +TString myRaw; +Proto2Json(msg, myRaw); +UNIT_ASSERT_STRINGS_EQUAL(myRaw, expRaw); + +myRaw = PrintInlined(msg, [](const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor*) { return false; }); +UNIT_ASSERT_STRINGS_EQUAL(myRaw, expRaw); // result is the same +} Y_UNIT_TEST(TestInliningPrinter) { - NProtobufJsonUt::TInlineTest msg = GetTestMsg(); - // inlined print should output these fields as inlined json sub-objects - TString expInlined = R"({"OptJson":{"a":1,"b":"000"},"NotJson":"12{}34","RepJson":[{},[1,2]],)" - R"("Inner":{"Number":[100,200],"InnerJson":{"xxx":[]}}})"; - - { - TString myInlined = PrintInlined(msg, MakeFieldOptionFunctor(NProtobufJsonUt::inline_test)); - UNIT_ASSERT_STRINGS_EQUAL(myInlined, expInlined); + NProtobufJsonUt::TInlineTest msg = GetTestMsg(); + // inlined print should output these fields as inlined json sub-objects + TString expInlined = R"({"OptJson":{"a":1,"b":"000"},"NotJson":"12{}34","RepJson":[{},[1,2]],)" + R"("Inner":{"Number":[100,200],"InnerJson":{"xxx":[]}}})"; + + { + TString myInlined = PrintInlined(msg, MakeFieldOptionFunctor(NProtobufJsonUt::inline_test)); + UNIT_ASSERT_STRINGS_EQUAL(myInlined, expInlined); } - { - auto functor = [](const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor* field) { - return field->name() == "OptJson" || field->name() == "RepJson" || field->name() == "InnerJson"; - }; - TString myInlined = PrintInlined(msg, functor); - UNIT_ASSERT_STRINGS_EQUAL(myInlined, expInlined); - } -} + { + auto functor = [](const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor* field) { + return field->name() == "OptJson" || field->name() == "RepJson" || field->name() == "InnerJson"; + }; + TString myInlined = PrintInlined(msg, functor); + UNIT_ASSERT_STRINGS_EQUAL(myInlined, expInlined); + } +} Y_UNIT_TEST(TestNoValues) { - // no values - no printing - NProtobufJsonUt::TInlineTest msg; - msg.MutableInner()->AddNumber(100); - msg.MutableInner()->AddNumber(200); + // no values - no printing + NProtobufJsonUt::TInlineTest msg; + msg.MutableInner()->AddNumber(100); + msg.MutableInner()->AddNumber(200); - TString expInlined = R"({"Inner":{"Number":[100,200]}})"; + TString expInlined = R"({"Inner":{"Number":[100,200]}})"; - TString myInlined = PrintInlined(msg, MakeFieldOptionFunctor(NProtobufJsonUt::inline_test)); - UNIT_ASSERT_STRINGS_EQUAL(myInlined, expInlined); -} + TString myInlined = PrintInlined(msg, MakeFieldOptionFunctor(NProtobufJsonUt::inline_test)); + UNIT_ASSERT_STRINGS_EQUAL(myInlined, expInlined); +} Y_UNIT_TEST(TestMissingKeyModeNull) { - NProtobufJsonUt::TInlineTest msg; - msg.MutableInner()->AddNumber(100); - msg.MutableInner()->AddNumber(200); + NProtobufJsonUt::TInlineTest msg; + msg.MutableInner()->AddNumber(100); + msg.MutableInner()->AddNumber(200); - TString expInlined = R"({"OptJson":null,"NotJson":null,"RepJson":null,"Inner":{"Number":[100,200],"InnerJson":null}})"; + TString expInlined = R"({"OptJson":null,"NotJson":null,"RepJson":null,"Inner":{"Number":[100,200],"InnerJson":null}})"; - TProto2JsonConfig cfg; - cfg.SetMissingSingleKeyMode(TProto2JsonConfig::MissingKeyNull).SetMissingRepeatedKeyMode(TProto2JsonConfig::MissingKeyNull); - TString myInlined = PrintInlined(msg, MakeFieldOptionFunctor(NProtobufJsonUt::inline_test), cfg); - UNIT_ASSERT_STRINGS_EQUAL(myInlined, expInlined); -} + TProto2JsonConfig cfg; + cfg.SetMissingSingleKeyMode(TProto2JsonConfig::MissingKeyNull).SetMissingRepeatedKeyMode(TProto2JsonConfig::MissingKeyNull); + TString myInlined = PrintInlined(msg, MakeFieldOptionFunctor(NProtobufJsonUt::inline_test), cfg); + UNIT_ASSERT_STRINGS_EQUAL(myInlined, expInlined); +} Y_UNIT_TEST(TestMissingKeyModeDefault) { - NProtobufJsonUt::TInlineTestDefaultValues msg; + NProtobufJsonUt::TInlineTestDefaultValues msg; - TString expInlined = R"({"OptJson":{"default":1},"Number":0,"RepJson":[],"Inner":{"OptJson":{"default":2}}})"; + TString expInlined = R"({"OptJson":{"default":1},"Number":0,"RepJson":[],"Inner":{"OptJson":{"default":2}}})"; - TProto2JsonConfig cfg; - cfg.SetMissingSingleKeyMode(TProto2JsonConfig::MissingKeyDefault).SetMissingRepeatedKeyMode(TProto2JsonConfig::MissingKeyDefault); - TString myInlined = PrintInlined(msg, MakeFieldOptionFunctor(NProtobufJsonUt::inline_test), cfg); - UNIT_ASSERT_STRINGS_EQUAL(myInlined, expInlined); -} + TProto2JsonConfig cfg; + cfg.SetMissingSingleKeyMode(TProto2JsonConfig::MissingKeyDefault).SetMissingRepeatedKeyMode(TProto2JsonConfig::MissingKeyDefault); + TString myInlined = PrintInlined(msg, MakeFieldOptionFunctor(NProtobufJsonUt::inline_test), cfg); + UNIT_ASSERT_STRINGS_EQUAL(myInlined, expInlined); +} Y_UNIT_TEST(NoUnnecessaryCopyFunctor) { - size_t CopyCount = 0; - struct TFunctorMock { - TFunctorMock(size_t* copyCount) - : CopyCount(copyCount) - { - UNIT_ASSERT(*CopyCount <= 1); - } - - TFunctorMock(const TFunctorMock& f) - : CopyCount(f.CopyCount) - { - ++*CopyCount; - } - - TFunctorMock(TFunctorMock&& f) = default; - - bool operator()(const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor*) const { - return false; - } - - size_t* CopyCount; - }; - - TProto2JsonConfig cfg; - TInliningPrinter<> printer(TFunctorMock(&CopyCount), cfg); - UNIT_ASSERT(CopyCount <= 1); -} -} -; + size_t CopyCount = 0; + struct TFunctorMock { + TFunctorMock(size_t* copyCount) + : CopyCount(copyCount) + { + UNIT_ASSERT(*CopyCount <= 1); + } + + TFunctorMock(const TFunctorMock& f) + : CopyCount(f.CopyCount) + { + ++*CopyCount; + } + + TFunctorMock(TFunctorMock&& f) = default; + + bool operator()(const NProtoBuf::Message&, const NProtoBuf::FieldDescriptor*) const { + return false; + } + + size_t* CopyCount; + }; + + TProto2JsonConfig cfg; + TInliningPrinter<> printer(TFunctorMock(&CopyCount), cfg); + UNIT_ASSERT(CopyCount <= 1); +} +} +; diff --git a/library/cpp/protobuf/json/ut/json.h b/library/cpp/protobuf/json/ut/json.h index c1f108e6e4..bdab69f4d4 100644 --- a/library/cpp/protobuf/json/ut/json.h +++ b/library/cpp/protobuf/json/ut/json.h @@ -12,58 +12,58 @@ #include <util/system/defaults.h> namespace NProtobufJsonTest { - inline NJson::TJsonValue - CreateFlatJson(const THashSet<TString>& skippedKeys = THashSet<TString>()) { - NJson::TJsonValue json; + inline NJson::TJsonValue + CreateFlatJson(const THashSet<TString>& skippedKeys = THashSet<TString>()) { + NJson::TJsonValue json; -#define DEFINE_FIELD(name, value) \ - if (skippedKeys.find(#name) == skippedKeys.end()) \ +#define DEFINE_FIELD(name, value) \ + if (skippedKeys.find(#name) == skippedKeys.end()) \ json.InsertValue(#name, value); #include "fields.incl" #undef DEFINE_FIELD - return json; - } - - inline NJson::TJsonValue - CreateRepeatedFlatJson(const THashSet<TString>& skippedKeys = THashSet<TString>()) { - NJson::TJsonValue json; - -#define DEFINE_REPEATED_FIELD(name, type, ...) \ - if (skippedKeys.find(#name) == skippedKeys.end()) { \ - type values[] = {__VA_ARGS__}; \ - NJson::TJsonValue array(NJson::JSON_ARRAY); \ - for (size_t i = 0, end = Y_ARRAY_SIZE(values); i < end; ++i) { \ - array.AppendValue(values[i]); \ - } \ - json.InsertValue(#name, array); \ + return json; + } + + inline NJson::TJsonValue + CreateRepeatedFlatJson(const THashSet<TString>& skippedKeys = THashSet<TString>()) { + NJson::TJsonValue json; + +#define DEFINE_REPEATED_FIELD(name, type, ...) \ + if (skippedKeys.find(#name) == skippedKeys.end()) { \ + type values[] = {__VA_ARGS__}; \ + NJson::TJsonValue array(NJson::JSON_ARRAY); \ + for (size_t i = 0, end = Y_ARRAY_SIZE(values); i < end; ++i) { \ + array.AppendValue(values[i]); \ + } \ + json.InsertValue(#name, array); \ } #include "repeated_fields.incl" #undef DEFINE_REPEATED_FIELD - return json; - } + return json; + } - inline NJson::TJsonValue - CreateCompositeJson(const THashSet<TString>& skippedKeys = THashSet<TString>()) { - const NJson::TJsonValue& part = CreateFlatJson(skippedKeys); - NJson::TJsonValue json; - json.InsertValue("Part", part); + inline NJson::TJsonValue + CreateCompositeJson(const THashSet<TString>& skippedKeys = THashSet<TString>()) { + const NJson::TJsonValue& part = CreateFlatJson(skippedKeys); + NJson::TJsonValue json; + json.InsertValue("Part", part); - return json; - } + return json; + } -#define UNIT_ASSERT_JSONS_EQUAL(lhs, rhs) \ - if (lhs != rhs) { \ +#define UNIT_ASSERT_JSONS_EQUAL(lhs, rhs) \ + if (lhs != rhs) { \ UNIT_ASSERT_STRINGS_EQUAL(lhs.GetStringRobust(), rhs.GetStringRobust()); \ } -#define UNIT_ASSERT_JSON_STRINGS_EQUAL(lhs, rhs) \ - if (lhs != rhs) { \ - NJson::TJsonValue _lhs_json, _rhs_json; \ +#define UNIT_ASSERT_JSON_STRINGS_EQUAL(lhs, rhs) \ + if (lhs != rhs) { \ + NJson::TJsonValue _lhs_json, _rhs_json; \ UNIT_ASSERT(NJson::ReadJsonTree(lhs, &_lhs_json)); \ UNIT_ASSERT(NJson::ReadJsonTree(rhs, &_rhs_json)); \ - UNIT_ASSERT_JSONS_EQUAL(_lhs_json, _rhs_json); \ + UNIT_ASSERT_JSONS_EQUAL(_lhs_json, _rhs_json); \ } -} +} diff --git a/library/cpp/protobuf/json/ut/json2proto_ut.cpp b/library/cpp/protobuf/json/ut/json2proto_ut.cpp index 0dfe57bc7a..c8fbcce65c 100644 --- a/library/cpp/protobuf/json/ut/json2proto_ut.cpp +++ b/library/cpp/protobuf/json/ut/json2proto_ut.cpp @@ -24,13 +24,13 @@ using namespace NProtobufJson; using namespace NProtobufJsonTest; namespace google { - namespace protobuf { - namespace internal { - void MapTestForceDeterministic() { - google::protobuf::io::CodedOutputStream::SetDefaultSerializationDeterministic(); - } - } - } // namespace protobuf + namespace protobuf { + namespace internal { + void MapTestForceDeterministic() { + google::protobuf::io::CodedOutputStream::SetDefaultSerializationDeterministic(); + } + } + } // namespace protobuf } namespace { @@ -72,331 +72,331 @@ namespace { Y_UNIT_TEST_SUITE(TJson2ProtoTest) { Y_UNIT_TEST(TestFlatOptional){ - {const NJson::TJsonValue& json = CreateFlatJson(); - TFlatOptional proto; - Json2Proto(json, proto); - TFlatOptional modelProto; - FillFlatProto(&modelProto); - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); -} - - // Try to skip each field -#define DEFINE_FIELD(name, value) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - const NJson::TJsonValue& json = CreateFlatJson(skippedField); \ - TFlatOptional proto; \ - Json2Proto(json, proto); \ - TFlatOptional modelProto; \ - FillFlatProto(&modelProto, skippedField); \ - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); \ - } + {const NJson::TJsonValue& json = CreateFlatJson(); + TFlatOptional proto; + Json2Proto(json, proto); + TFlatOptional modelProto; + FillFlatProto(&modelProto); + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); +} + + // Try to skip each field +#define DEFINE_FIELD(name, value) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + const NJson::TJsonValue& json = CreateFlatJson(skippedField); \ + TFlatOptional proto; \ + Json2Proto(json, proto); \ + TFlatOptional modelProto; \ + FillFlatProto(&modelProto, skippedField); \ + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); \ + } #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD -} // TestFlatOptional +} // TestFlatOptional Y_UNIT_TEST(TestFlatRequired){ - {const NJson::TJsonValue& json = CreateFlatJson(); -TFlatRequired proto; -Json2Proto(json, proto); -TFlatRequired modelProto; -FillFlatProto(&modelProto); -UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); -} - -// Try to skip each field -#define DEFINE_FIELD(name, value) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - const NJson::TJsonValue& json = CreateFlatJson(skippedField); \ - TFlatRequired proto; \ - UNIT_ASSERT_EXCEPTION(Json2Proto(json, proto), yexception); \ - } + {const NJson::TJsonValue& json = CreateFlatJson(); +TFlatRequired proto; +Json2Proto(json, proto); +TFlatRequired modelProto; +FillFlatProto(&modelProto); +UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); +} + +// Try to skip each field +#define DEFINE_FIELD(name, value) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + const NJson::TJsonValue& json = CreateFlatJson(skippedField); \ + TFlatRequired proto; \ + UNIT_ASSERT_EXCEPTION(Json2Proto(json, proto), yexception); \ + } #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD -} // TestFlatRequired +} // TestFlatRequired Y_UNIT_TEST(TestNameGenerator) { - TJson2ProtoConfig cfg; - cfg.SetNameGenerator([](const NProtoBuf::FieldDescriptor&) { return "42"; }); + TJson2ProtoConfig cfg; + cfg.SetNameGenerator([](const NProtoBuf::FieldDescriptor&) { return "42"; }); - TNameGeneratorType proto; + TNameGeneratorType proto; Json2Proto(TStringBuf(R"({"42":42})"), proto, cfg); - TNameGeneratorType expected; - expected.SetField(42); - - UNIT_ASSERT_PROTOS_EQUAL(expected, proto); -} + TNameGeneratorType expected; + expected.SetField(42); + UNIT_ASSERT_PROTOS_EQUAL(expected, proto); +} + Y_UNIT_TEST(TestFlatNoCheckRequired) { - { - const NJson::TJsonValue& json = CreateFlatJson(); - TFlatRequired proto; - Json2Proto(json, proto); - TFlatRequired modelProto; - FillFlatProto(&modelProto); - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); + { + const NJson::TJsonValue& json = CreateFlatJson(); + TFlatRequired proto; + Json2Proto(json, proto); + TFlatRequired modelProto; + FillFlatProto(&modelProto); + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); } - TJson2ProtoConfig cfg; - cfg.CheckRequiredFields = false; - - // Try to skip each field -#define DEFINE_FIELD(name, value) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - const NJson::TJsonValue& json = CreateFlatJson(skippedField); \ - TFlatRequired proto; \ - UNIT_ASSERT_NO_EXCEPTION(Json2Proto(json, proto, cfg)); \ - } + TJson2ProtoConfig cfg; + cfg.CheckRequiredFields = false; + + // Try to skip each field +#define DEFINE_FIELD(name, value) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + const NJson::TJsonValue& json = CreateFlatJson(skippedField); \ + TFlatRequired proto; \ + UNIT_ASSERT_NO_EXCEPTION(Json2Proto(json, proto, cfg)); \ + } #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD -} // TestFlatNoCheckRequired +} // TestFlatNoCheckRequired Y_UNIT_TEST(TestFlatRepeated){ - {const NJson::TJsonValue& json = CreateRepeatedFlatJson(); -TFlatRepeated proto; -Json2Proto(json, proto); -TFlatRepeated modelProto; -FillRepeatedProto(&modelProto); -UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); -} - -// Try to skip each field -#define DEFINE_REPEATED_FIELD(name, ...) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - const NJson::TJsonValue& json = CreateRepeatedFlatJson(skippedField); \ - TFlatRepeated proto; \ - Json2Proto(json, proto); \ - TFlatRepeated modelProto; \ - FillRepeatedProto(&modelProto, skippedField); \ - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); \ - } + {const NJson::TJsonValue& json = CreateRepeatedFlatJson(); +TFlatRepeated proto; +Json2Proto(json, proto); +TFlatRepeated modelProto; +FillRepeatedProto(&modelProto); +UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); +} + +// Try to skip each field +#define DEFINE_REPEATED_FIELD(name, ...) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + const NJson::TJsonValue& json = CreateRepeatedFlatJson(skippedField); \ + TFlatRepeated proto; \ + Json2Proto(json, proto); \ + TFlatRepeated modelProto; \ + FillRepeatedProto(&modelProto, skippedField); \ + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); \ + } #include <library/cpp/protobuf/json/ut/repeated_fields.incl> #undef DEFINE_REPEATED_FIELD -} // TestFlatRepeated +} // TestFlatRepeated Y_UNIT_TEST(TestCompositeOptional){ - {const NJson::TJsonValue& json = CreateCompositeJson(); -TCompositeOptional proto; -Json2Proto(json, proto); -TCompositeOptional modelProto; -FillCompositeProto(&modelProto); -UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); -} - -// Try to skip each field -#define DEFINE_FIELD(name, value) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - const NJson::TJsonValue& json = CreateCompositeJson(skippedField); \ - TCompositeOptional proto; \ - Json2Proto(json, proto); \ - TCompositeOptional modelProto; \ - FillCompositeProto(&modelProto, skippedField); \ - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); \ - } + {const NJson::TJsonValue& json = CreateCompositeJson(); +TCompositeOptional proto; +Json2Proto(json, proto); +TCompositeOptional modelProto; +FillCompositeProto(&modelProto); +UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); +} + +// Try to skip each field +#define DEFINE_FIELD(name, value) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + const NJson::TJsonValue& json = CreateCompositeJson(skippedField); \ + TCompositeOptional proto; \ + Json2Proto(json, proto); \ + TCompositeOptional modelProto; \ + FillCompositeProto(&modelProto, skippedField); \ + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); \ + } #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD -} // TestCompositeOptional +} // TestCompositeOptional Y_UNIT_TEST(TestCompositeOptionalStringBuf){ - {NJson::TJsonValue json = CreateCompositeJson(); -json["Part"]["Double"] = 42.5; -TCompositeOptional proto; -Json2Proto(JsonValueToString(json), proto); -TCompositeOptional modelProto; -FillCompositeProto(&modelProto); -modelProto.MutablePart()->SetDouble(42.5); -UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); -} - -// Try to skip each field -#define DEFINE_FIELD(name, value) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - NJson::TJsonValue json = CreateCompositeJson(skippedField); \ - if (json["Part"].Has("Double")) { \ - json["Part"]["Double"] = 42.5; \ - } \ - TCompositeOptional proto; \ - Json2Proto(JsonValueToString(json), proto); \ - TCompositeOptional modelProto; \ - FillCompositeProto(&modelProto, skippedField); \ - if (modelProto.GetPart().HasDouble()) { \ - modelProto.MutablePart()->SetDouble(42.5); \ - } \ - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); \ - } + {NJson::TJsonValue json = CreateCompositeJson(); +json["Part"]["Double"] = 42.5; +TCompositeOptional proto; +Json2Proto(JsonValueToString(json), proto); +TCompositeOptional modelProto; +FillCompositeProto(&modelProto); +modelProto.MutablePart()->SetDouble(42.5); +UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); +} + +// Try to skip each field +#define DEFINE_FIELD(name, value) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + NJson::TJsonValue json = CreateCompositeJson(skippedField); \ + if (json["Part"].Has("Double")) { \ + json["Part"]["Double"] = 42.5; \ + } \ + TCompositeOptional proto; \ + Json2Proto(JsonValueToString(json), proto); \ + TCompositeOptional modelProto; \ + FillCompositeProto(&modelProto, skippedField); \ + if (modelProto.GetPart().HasDouble()) { \ + modelProto.MutablePart()->SetDouble(42.5); \ + } \ + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); \ + } #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD -} // TestCompositeOptionalStringBuf +} // TestCompositeOptionalStringBuf Y_UNIT_TEST(TestCompositeRequired) { - { - const NJson::TJsonValue& json = CreateCompositeJson(); - TCompositeRequired proto; - Json2Proto(json, proto); - TCompositeRequired modelProto; - FillCompositeProto(&modelProto); - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); - } - - { - NJson::TJsonValue json; - TCompositeRequired proto; - UNIT_ASSERT_EXCEPTION(Json2Proto(json, proto), yexception); - } -} // TestCompositeRequired + { + const NJson::TJsonValue& json = CreateCompositeJson(); + TCompositeRequired proto; + Json2Proto(json, proto); + TCompositeRequired modelProto; + FillCompositeProto(&modelProto); + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); + } + + { + NJson::TJsonValue json; + TCompositeRequired proto; + UNIT_ASSERT_EXCEPTION(Json2Proto(json, proto), yexception); + } +} // TestCompositeRequired Y_UNIT_TEST(TestCompositeRepeated) { - { - NJson::TJsonValue json; - NJson::TJsonValue array; - array.AppendValue(CreateFlatJson()); - json.InsertValue("Part", array); - - TCompositeRepeated proto; - Json2Proto(json, proto); - - TFlatOptional partModelProto; - FillFlatProto(&partModelProto); - TCompositeRepeated modelProto; - modelProto.AddPart()->CopyFrom(partModelProto); - - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); - } - - { - // Array of messages with each field skipped - TCompositeRepeated modelProto; - NJson::TJsonValue array; - -#define DEFINE_REPEATED_FIELD(name, ...) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - TFlatOptional partModelProto; \ - FillFlatProto(&partModelProto, skippedField); \ - modelProto.AddPart()->CopyFrom(partModelProto); \ - array.AppendValue(CreateFlatJson(skippedField)); \ - } + { + NJson::TJsonValue json; + NJson::TJsonValue array; + array.AppendValue(CreateFlatJson()); + json.InsertValue("Part", array); + + TCompositeRepeated proto; + Json2Proto(json, proto); + + TFlatOptional partModelProto; + FillFlatProto(&partModelProto); + TCompositeRepeated modelProto; + modelProto.AddPart()->CopyFrom(partModelProto); + + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); + } + + { + // Array of messages with each field skipped + TCompositeRepeated modelProto; + NJson::TJsonValue array; + +#define DEFINE_REPEATED_FIELD(name, ...) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + TFlatOptional partModelProto; \ + FillFlatProto(&partModelProto, skippedField); \ + modelProto.AddPart()->CopyFrom(partModelProto); \ + array.AppendValue(CreateFlatJson(skippedField)); \ + } #include <library/cpp/protobuf/json/ut/repeated_fields.incl> #undef DEFINE_REPEATED_FIELD - NJson::TJsonValue json; - json.InsertValue("Part", array); + NJson::TJsonValue json; + json.InsertValue("Part", array); - TCompositeRepeated proto; - Json2Proto(json, proto); + TCompositeRepeated proto; + Json2Proto(json, proto); - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); - } -} // TestCompositeRepeated + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); + } +} // TestCompositeRepeated Y_UNIT_TEST(TestInvalidEnum) { - { - NJson::TJsonValue json; - json.InsertValue("Enum", "E_100"); - TFlatOptional proto; - UNIT_ASSERT_EXCEPTION(Json2Proto(json, proto), yexception); - } - - { - NJson::TJsonValue json; - json.InsertValue("Enum", 100); - TFlatOptional proto; - UNIT_ASSERT_EXCEPTION(Json2Proto(json, proto), yexception); + { + NJson::TJsonValue json; + json.InsertValue("Enum", "E_100"); + TFlatOptional proto; + UNIT_ASSERT_EXCEPTION(Json2Proto(json, proto), yexception); + } + + { + NJson::TJsonValue json; + json.InsertValue("Enum", 100); + TFlatOptional proto; + UNIT_ASSERT_EXCEPTION(Json2Proto(json, proto), yexception); } -} +} Y_UNIT_TEST(TestFieldNameMode) { - // Original case 1 - { - TString modelStr(R"_({"String":"value"})_"); + // Original case 1 + { + TString modelStr(R"_({"String":"value"})_"); - TFlatOptional proto; - TJson2ProtoConfig config; + TFlatOptional proto; + TJson2ProtoConfig config; - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); - UNIT_ASSERT(proto.GetString() == "value"); - } + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); + UNIT_ASSERT(proto.GetString() == "value"); + } - // Original case 2 - { - TString modelStr(R"_({"String":"value"})_"); + // Original case 2 + { + TString modelStr(R"_({"String":"value"})_"); - TFlatOptional proto; - TJson2ProtoConfig config; - config.FieldNameMode = TJson2ProtoConfig::FieldNameOriginalCase; + TFlatOptional proto; + TJson2ProtoConfig config; + config.FieldNameMode = TJson2ProtoConfig::FieldNameOriginalCase; - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); - UNIT_ASSERT(proto.GetString() == "value"); - } + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); + UNIT_ASSERT(proto.GetString() == "value"); + } - // Lowercase - { - TString modelStr(R"_({"string":"value"})_"); + // Lowercase + { + TString modelStr(R"_({"string":"value"})_"); - TFlatOptional proto; - TJson2ProtoConfig config; - config.FieldNameMode = TJson2ProtoConfig::FieldNameLowerCase; + TFlatOptional proto; + TJson2ProtoConfig config; + config.FieldNameMode = TJson2ProtoConfig::FieldNameLowerCase; - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); - UNIT_ASSERT(proto.GetString() == "value"); - } + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); + UNIT_ASSERT(proto.GetString() == "value"); + } - // Uppercase - { - TString modelStr(R"_({"STRING":"value"})_"); + // Uppercase + { + TString modelStr(R"_({"STRING":"value"})_"); - TFlatOptional proto; - TJson2ProtoConfig config; - config.FieldNameMode = TJson2ProtoConfig::FieldNameUpperCase; + TFlatOptional proto; + TJson2ProtoConfig config; + config.FieldNameMode = TJson2ProtoConfig::FieldNameUpperCase; - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); - UNIT_ASSERT(proto.GetString() == "value"); - } + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); + UNIT_ASSERT(proto.GetString() == "value"); + } - // Camelcase - { - TString modelStr(R"_({"string":"value"})_"); + // Camelcase + { + TString modelStr(R"_({"string":"value"})_"); - TFlatOptional proto; - TJson2ProtoConfig config; - config.FieldNameMode = TJson2ProtoConfig::FieldNameCamelCase; + TFlatOptional proto; + TJson2ProtoConfig config; + config.FieldNameMode = TJson2ProtoConfig::FieldNameCamelCase; - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); - UNIT_ASSERT(proto.GetString() == "value"); - } - { - TString modelStr(R"_({"oneString":"value"})_"); + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); + UNIT_ASSERT(proto.GetString() == "value"); + } + { + TString modelStr(R"_({"oneString":"value"})_"); - TFlatOptional proto; - TJson2ProtoConfig config; - config.FieldNameMode = TJson2ProtoConfig::FieldNameCamelCase; + TFlatOptional proto; + TJson2ProtoConfig config; + config.FieldNameMode = TJson2ProtoConfig::FieldNameCamelCase; - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); - UNIT_ASSERT(proto.GetOneString() == "value"); - } - { - TString modelStr(R"_({"oneTwoString":"value"})_"); + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); + UNIT_ASSERT(proto.GetOneString() == "value"); + } + { + TString modelStr(R"_({"oneTwoString":"value"})_"); - TFlatOptional proto; - TJson2ProtoConfig config; - config.FieldNameMode = TJson2ProtoConfig::FieldNameCamelCase; + TFlatOptional proto; + TJson2ProtoConfig config; + config.FieldNameMode = TJson2ProtoConfig::FieldNameCamelCase; - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); - UNIT_ASSERT(proto.GetOneTwoString() == "value"); - } + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); + UNIT_ASSERT(proto.GetOneTwoString() == "value"); + } // snake_case { @@ -430,95 +430,95 @@ Y_UNIT_TEST(TestFieldNameMode) { UNIT_ASSERT(proto.GetOneTwoString() == "value"); } - // Original case, repeated - { - TString modelStr(R"_({"I32":[1,2]})_"); - - TFlatRepeated proto; - TJson2ProtoConfig config; - config.FieldNameMode = TJson2ProtoConfig::FieldNameOriginalCase; - - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatRepeated>(modelStr, config)); - UNIT_ASSERT(proto.I32Size() == 2); - UNIT_ASSERT(proto.GetI32(0) == 1); - UNIT_ASSERT(proto.GetI32(1) == 2); - } - - // Lower case, repeated - { - TString modelStr(R"_({"i32":[1,2]})_"); - - TFlatRepeated proto; - TJson2ProtoConfig config; - config.FieldNameMode = TJson2ProtoConfig::FieldNameLowerCase; - - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatRepeated>(modelStr, config)); - UNIT_ASSERT(proto.I32Size() == 2); - UNIT_ASSERT(proto.GetI32(0) == 1); - UNIT_ASSERT(proto.GetI32(1) == 2); - } - - // UseJsonName - { - // FIXME(CONTRIB-139): since protobuf 3.1, Def_upper json name is - // "DefUpper", but until kernel/ugc/schema and yweb/yasap/pdb are + // Original case, repeated + { + TString modelStr(R"_({"I32":[1,2]})_"); + + TFlatRepeated proto; + TJson2ProtoConfig config; + config.FieldNameMode = TJson2ProtoConfig::FieldNameOriginalCase; + + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatRepeated>(modelStr, config)); + UNIT_ASSERT(proto.I32Size() == 2); + UNIT_ASSERT(proto.GetI32(0) == 1); + UNIT_ASSERT(proto.GetI32(1) == 2); + } + + // Lower case, repeated + { + TString modelStr(R"_({"i32":[1,2]})_"); + + TFlatRepeated proto; + TJson2ProtoConfig config; + config.FieldNameMode = TJson2ProtoConfig::FieldNameLowerCase; + + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatRepeated>(modelStr, config)); + UNIT_ASSERT(proto.I32Size() == 2); + UNIT_ASSERT(proto.GetI32(0) == 1); + UNIT_ASSERT(proto.GetI32(1) == 2); + } + + // UseJsonName + { + // FIXME(CONTRIB-139): since protobuf 3.1, Def_upper json name is + // "DefUpper", but until kernel/ugc/schema and yweb/yasap/pdb are // updated, library/cpp/protobuf/json preserves compatibility with - // protobuf 3.0 by lowercasing default names, making it "defUpper". - TString modelStr(R"_({"My-Upper":1,"my-lower":2,"defUpper":3,"defLower":4})_"); - - TWithJsonName proto; - TJson2ProtoConfig config; - config.SetUseJsonName(true); - - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TWithJsonName>(modelStr, config)); - UNIT_ASSERT_EQUAL(proto.Getmy_upper(), 1); - UNIT_ASSERT_EQUAL(proto.GetMy_lower(), 2); - UNIT_ASSERT_EQUAL(proto.GetDef_upper(), 3); - UNIT_ASSERT_EQUAL(proto.Getdef_lower(), 4); - } - - // FieldNameMode with UseJsonName - { - TJson2ProtoConfig config; - config.SetFieldNameMode(TJson2ProtoConfig::FieldNameLowerCase); - UNIT_ASSERT_EXCEPTION_CONTAINS( - config.SetUseJsonName(true), yexception, "mutually exclusive"); - } - { - TJson2ProtoConfig config; - config.SetUseJsonName(true); - UNIT_ASSERT_EXCEPTION_CONTAINS( - config.SetFieldNameMode(TJson2ProtoConfig::FieldNameLowerCase), yexception, "mutually exclusive"); - } -} // TestFieldNameMode - -class TStringTransform: public IStringTransform { -public: - int GetType() const override { - return 0; + // protobuf 3.0 by lowercasing default names, making it "defUpper". + TString modelStr(R"_({"My-Upper":1,"my-lower":2,"defUpper":3,"defLower":4})_"); + + TWithJsonName proto; + TJson2ProtoConfig config; + config.SetUseJsonName(true); + + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TWithJsonName>(modelStr, config)); + UNIT_ASSERT_EQUAL(proto.Getmy_upper(), 1); + UNIT_ASSERT_EQUAL(proto.GetMy_lower(), 2); + UNIT_ASSERT_EQUAL(proto.GetDef_upper(), 3); + UNIT_ASSERT_EQUAL(proto.Getdef_lower(), 4); + } + + // FieldNameMode with UseJsonName + { + TJson2ProtoConfig config; + config.SetFieldNameMode(TJson2ProtoConfig::FieldNameLowerCase); + UNIT_ASSERT_EXCEPTION_CONTAINS( + config.SetUseJsonName(true), yexception, "mutually exclusive"); + } + { + TJson2ProtoConfig config; + config.SetUseJsonName(true); + UNIT_ASSERT_EXCEPTION_CONTAINS( + config.SetFieldNameMode(TJson2ProtoConfig::FieldNameLowerCase), yexception, "mutually exclusive"); + } +} // TestFieldNameMode + +class TStringTransform: public IStringTransform { +public: + int GetType() const override { + return 0; + } + void Transform(TString& str) const override { + str = "transformed_any"; + } +}; + +class TBytesTransform: public IStringTransform { +public: + int GetType() const override { + return 0; } - void Transform(TString& str) const override { - str = "transformed_any"; - } -}; - -class TBytesTransform: public IStringTransform { -public: - int GetType() const override { - return 0; - } - void Transform(TString&) const override { - } - void TransformBytes(TString& str) const override { - str = "transformed_bytes"; - } -}; + void Transform(TString&) const override { + } + void TransformBytes(TString& str) const override { + str = "transformed_bytes"; + } +}; Y_UNIT_TEST(TestInvalidJson) { - NJson::TJsonValue val{"bad value"}; - TFlatOptional proto; - UNIT_ASSERT_EXCEPTION(Json2Proto(val, proto), yexception); -} + NJson::TJsonValue val{"bad value"}; + TFlatOptional proto; + UNIT_ASSERT_EXCEPTION(Json2Proto(val, proto), yexception); +} Y_UNIT_TEST(TestInvalidRepeatedFieldWithMapAsObject) { TCompositeRepeated proto; @@ -528,149 +528,149 @@ Y_UNIT_TEST(TestInvalidRepeatedFieldWithMapAsObject) { } Y_UNIT_TEST(TestStringTransforms) { - // Check that strings and bytes are transformed - { - TString modelStr(R"_({"String":"value_str", "Bytes": "value_bytes"})_"); - - TFlatOptional proto; - TJson2ProtoConfig config; - config.AddStringTransform(new TStringTransform); - - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); - UNIT_ASSERT(proto.GetString() == "transformed_any"); - UNIT_ASSERT(proto.GetBytes() == "transformed_any"); - } - - // Check that bytes are transformed, strings are left intact - { - TString modelStr(R"_({"String":"value_str", "Bytes": "value_bytes"})_"); - - TFlatOptional proto; - TJson2ProtoConfig config; - config.AddStringTransform(new TBytesTransform); - - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); - UNIT_ASSERT(proto.GetString() == "value_str"); - UNIT_ASSERT(proto.GetBytes() == "transformed_bytes"); - } - - // Check that repeated bytes are transformed, repeated strings are left intact - { - TString modelStr(R"_({"String":["value_str", "str2"], "Bytes": ["value_bytes", "bytes2"]})_"); - - TFlatRepeated proto; - TJson2ProtoConfig config; - config.AddStringTransform(new TBytesTransform); - - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatRepeated>(modelStr, config)); - UNIT_ASSERT(proto.StringSize() == 2); - UNIT_ASSERT(proto.GetString(0) == "value_str"); - UNIT_ASSERT(proto.GetString(1) == "str2"); - UNIT_ASSERT(proto.BytesSize() == 2); - UNIT_ASSERT(proto.GetBytes(0) == "transformed_bytes"); - UNIT_ASSERT(proto.GetBytes(1) == "transformed_bytes"); - } - - // Check that bytes are transformed, strings are left intact in composed messages - { - TString modelStr(R"_({"Part": {"String":"value_str", "Bytes": "value_bytes"}})_"); - - TCompositeOptional proto; - TJson2ProtoConfig config; - config.AddStringTransform(new TBytesTransform); - - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TCompositeOptional>(modelStr, config)); - UNIT_ASSERT(proto.GetPart().GetString() == "value_str"); - UNIT_ASSERT(proto.GetPart().GetBytes() == "transformed_bytes"); - } -} // TestStringTransforms - + // Check that strings and bytes are transformed + { + TString modelStr(R"_({"String":"value_str", "Bytes": "value_bytes"})_"); + + TFlatOptional proto; + TJson2ProtoConfig config; + config.AddStringTransform(new TStringTransform); + + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); + UNIT_ASSERT(proto.GetString() == "transformed_any"); + UNIT_ASSERT(proto.GetBytes() == "transformed_any"); + } + + // Check that bytes are transformed, strings are left intact + { + TString modelStr(R"_({"String":"value_str", "Bytes": "value_bytes"})_"); + + TFlatOptional proto; + TJson2ProtoConfig config; + config.AddStringTransform(new TBytesTransform); + + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatOptional>(modelStr, config)); + UNIT_ASSERT(proto.GetString() == "value_str"); + UNIT_ASSERT(proto.GetBytes() == "transformed_bytes"); + } + + // Check that repeated bytes are transformed, repeated strings are left intact + { + TString modelStr(R"_({"String":["value_str", "str2"], "Bytes": ["value_bytes", "bytes2"]})_"); + + TFlatRepeated proto; + TJson2ProtoConfig config; + config.AddStringTransform(new TBytesTransform); + + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TFlatRepeated>(modelStr, config)); + UNIT_ASSERT(proto.StringSize() == 2); + UNIT_ASSERT(proto.GetString(0) == "value_str"); + UNIT_ASSERT(proto.GetString(1) == "str2"); + UNIT_ASSERT(proto.BytesSize() == 2); + UNIT_ASSERT(proto.GetBytes(0) == "transformed_bytes"); + UNIT_ASSERT(proto.GetBytes(1) == "transformed_bytes"); + } + + // Check that bytes are transformed, strings are left intact in composed messages + { + TString modelStr(R"_({"Part": {"String":"value_str", "Bytes": "value_bytes"}})_"); + + TCompositeOptional proto; + TJson2ProtoConfig config; + config.AddStringTransform(new TBytesTransform); + + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TCompositeOptional>(modelStr, config)); + UNIT_ASSERT(proto.GetPart().GetString() == "value_str"); + UNIT_ASSERT(proto.GetPart().GetBytes() == "transformed_bytes"); + } +} // TestStringTransforms + Y_UNIT_TEST(TestCastFromString) { - // single fields - { - NJson::TJsonValue json; + // single fields + { + NJson::TJsonValue json; #define DEFINE_FIELD(name, value) \ - json.InsertValue(#name, ConvertToString(value)); + json.InsertValue(#name, ConvertToString(value)); #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD - TFlatOptional proto; - UNIT_ASSERT_EXCEPTION_CONTAINS(Json2Proto(json, proto), yexception, "Invalid type"); - - TJson2ProtoConfig config; - config.SetCastFromString(true); - Json2Proto(json, proto, config); - - TFlatOptional modelProto; - FillFlatProto(&modelProto); - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); - } - - // repeated fields - { - NJson::TJsonValue json; -#define DEFINE_REPEATED_FIELD(name, type, ...) \ - { \ - type values[] = {__VA_ARGS__}; \ - NJson::TJsonValue array(NJson::JSON_ARRAY); \ - for (size_t i = 0, end = Y_ARRAY_SIZE(values); i < end; ++i) { \ - array.AppendValue(ConvertToString(values[i])); \ - } \ - json.InsertValue(#name, array); \ - } + TFlatOptional proto; + UNIT_ASSERT_EXCEPTION_CONTAINS(Json2Proto(json, proto), yexception, "Invalid type"); + + TJson2ProtoConfig config; + config.SetCastFromString(true); + Json2Proto(json, proto, config); + + TFlatOptional modelProto; + FillFlatProto(&modelProto); + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); + } + + // repeated fields + { + NJson::TJsonValue json; +#define DEFINE_REPEATED_FIELD(name, type, ...) \ + { \ + type values[] = {__VA_ARGS__}; \ + NJson::TJsonValue array(NJson::JSON_ARRAY); \ + for (size_t i = 0, end = Y_ARRAY_SIZE(values); i < end; ++i) { \ + array.AppendValue(ConvertToString(values[i])); \ + } \ + json.InsertValue(#name, array); \ + } #include <library/cpp/protobuf/json/ut/repeated_fields.incl> #undef DEFINE_REPEATED_FIELD - TFlatRepeated proto; - UNIT_ASSERT_EXCEPTION_CONTAINS(Json2Proto(json, proto), yexception, "Invalid type"); + TFlatRepeated proto; + UNIT_ASSERT_EXCEPTION_CONTAINS(Json2Proto(json, proto), yexception, "Invalid type"); - TJson2ProtoConfig config; - config.SetCastFromString(true); - Json2Proto(json, proto, config); + TJson2ProtoConfig config; + config.SetCastFromString(true); + Json2Proto(json, proto, config); - TFlatRepeated modelProto; - FillRepeatedProto(&modelProto); - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); - } -} // TestCastFromString + TFlatRepeated modelProto; + FillRepeatedProto(&modelProto); + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); + } +} // TestCastFromString Y_UNIT_TEST(TestMap) { - TMapType modelProto; + TMapType modelProto; - auto& items = *modelProto.MutableItems(); - items["key1"] = "value1"; - items["key2"] = "value2"; - items["key3"] = "value3"; + auto& items = *modelProto.MutableItems(); + items["key1"] = "value1"; + items["key2"] = "value2"; + items["key3"] = "value3"; - TString modelStr(R"_({"Items":[{"key":"key3","value":"value3"},{"key":"key2","value":"value2"},{"key":"key1","value":"value1"}]})_"); + TString modelStr(R"_({"Items":[{"key":"key3","value":"value3"},{"key":"key2","value":"value2"},{"key":"key1","value":"value1"}]})_"); - TJson2ProtoConfig config; - TMapType proto; - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TMapType>(modelStr, config)); + TJson2ProtoConfig config; + TMapType proto; + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TMapType>(modelStr, config)); - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); -} // TestMap + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); +} // TestMap Y_UNIT_TEST(TestCastRobust) { - NJson::TJsonValue json; - json["I32"] = "5"; - json["Bool"] = 1; - json["String"] = 6; - json["Double"] = 8; - TFlatOptional proto; - UNIT_ASSERT_EXCEPTION_CONTAINS(Json2Proto(json, proto), yexception, "Invalid type"); - - TJson2ProtoConfig config; - config.SetCastRobust(true); - Json2Proto(json, proto, config); - - TFlatOptional expected; - expected.SetI32(5); - expected.SetBool(true); - expected.SetString("6"); - expected.SetDouble(8); - UNIT_ASSERT_PROTOS_EQUAL(proto, expected); -} + NJson::TJsonValue json; + json["I32"] = "5"; + json["Bool"] = 1; + json["String"] = 6; + json["Double"] = 8; + TFlatOptional proto; + UNIT_ASSERT_EXCEPTION_CONTAINS(Json2Proto(json, proto), yexception, "Invalid type"); + + TJson2ProtoConfig config; + config.SetCastRobust(true); + Json2Proto(json, proto, config); + + TFlatOptional expected; + expected.SetI32(5); + expected.SetBool(true); + expected.SetString("6"); + expected.SetDouble(8); + UNIT_ASSERT_PROTOS_EQUAL(proto, expected); +} Y_UNIT_TEST(TestVectorizeScalars) { NJson::TJsonValue json; @@ -742,165 +742,165 @@ Y_UNIT_TEST(TestValueVectorizer) { } Y_UNIT_TEST(TestMapAsObject) { - TMapType modelProto; + TMapType modelProto; - auto& items = *modelProto.MutableItems(); - items["key1"] = "value1"; - items["key2"] = "value2"; - items["key3"] = "value3"; + auto& items = *modelProto.MutableItems(); + items["key1"] = "value1"; + items["key2"] = "value2"; + items["key3"] = "value3"; - TString modelStr(R"_({"Items":{"key1":"value1","key2":"value2","key3":"value3"}})_"); + TString modelStr(R"_({"Items":{"key1":"value1","key2":"value2","key3":"value3"}})_"); - TJson2ProtoConfig config; - config.MapAsObject = true; - TMapType proto; - UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TMapType>(modelStr, config)); + TJson2ProtoConfig config; + config.MapAsObject = true; + TMapType proto; + UNIT_ASSERT_NO_EXCEPTION(proto = Json2Proto<TMapType>(modelStr, config)); - UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); -} // TestMapAsObject + UNIT_ASSERT_PROTOS_EQUAL(proto, modelProto); +} // TestMapAsObject Y_UNIT_TEST(TestComplexMapAsObject_I32) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableI32(); - items[1] = 1; - items[-2] = -2; - items[3] = 3; - }, - R"_({"I32":{"1":1,"-2":-2,"3":3}})_"); -} // TestComplexMapAsObject_I32 + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableI32(); + items[1] = 1; + items[-2] = -2; + items[3] = 3; + }, + R"_({"I32":{"1":1,"-2":-2,"3":3}})_"); +} // TestComplexMapAsObject_I32 Y_UNIT_TEST(TestComplexMapAsObject_I64) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableI64(); - items[2147483649L] = 2147483649L; - items[-2147483650L] = -2147483650L; - items[2147483651L] = 2147483651L; - }, - R"_({"I64":{"2147483649":2147483649,"-2147483650":-2147483650,"2147483651":2147483651}})_"); -} // TestComplexMapAsObject_I64 + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableI64(); + items[2147483649L] = 2147483649L; + items[-2147483650L] = -2147483650L; + items[2147483651L] = 2147483651L; + }, + R"_({"I64":{"2147483649":2147483649,"-2147483650":-2147483650,"2147483651":2147483651}})_"); +} // TestComplexMapAsObject_I64 Y_UNIT_TEST(TestComplexMapAsObject_UI32) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableUI32(); - items[1073741825U] = 1073741825U; - items[1073741826U] = 1073741826U; - items[1073741827U] = 1073741827U; - }, - R"_({"UI32":{"1073741825":1073741825,"1073741826":1073741826,"1073741827":1073741827}})_"); -} // TestComplexMapAsObject_UI32 + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableUI32(); + items[1073741825U] = 1073741825U; + items[1073741826U] = 1073741826U; + items[1073741827U] = 1073741827U; + }, + R"_({"UI32":{"1073741825":1073741825,"1073741826":1073741826,"1073741827":1073741827}})_"); +} // TestComplexMapAsObject_UI32 Y_UNIT_TEST(TestComplexMapAsObject_UI64) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableUI64(); - items[9223372036854775809UL] = 9223372036854775809UL; - items[9223372036854775810UL] = 9223372036854775810UL; - items[9223372036854775811UL] = 9223372036854775811UL; - }, - R"_({"UI64":{"9223372036854775809":9223372036854775809,"9223372036854775810":9223372036854775810,"9223372036854775811":9223372036854775811}})_"); -} // TestComplexMapAsObject_UI64 + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableUI64(); + items[9223372036854775809UL] = 9223372036854775809UL; + items[9223372036854775810UL] = 9223372036854775810UL; + items[9223372036854775811UL] = 9223372036854775811UL; + }, + R"_({"UI64":{"9223372036854775809":9223372036854775809,"9223372036854775810":9223372036854775810,"9223372036854775811":9223372036854775811}})_"); +} // TestComplexMapAsObject_UI64 Y_UNIT_TEST(TestComplexMapAsObject_SI32) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableSI32(); - items[1] = 1; - items[-2] = -2; - items[3] = 3; - }, - R"_({"SI32":{"1":1,"-2":-2,"3":3}})_"); -} // TestComplexMapAsObject_SI32 + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableSI32(); + items[1] = 1; + items[-2] = -2; + items[3] = 3; + }, + R"_({"SI32":{"1":1,"-2":-2,"3":3}})_"); +} // TestComplexMapAsObject_SI32 Y_UNIT_TEST(TestComplexMapAsObject_SI64) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableSI64(); - items[2147483649L] = 2147483649L; - items[-2147483650L] = -2147483650L; - items[2147483651L] = 2147483651L; - }, - R"_({"SI64":{"2147483649":2147483649,"-2147483650":-2147483650,"2147483651":2147483651}})_"); -} // TestComplexMapAsObject_SI64 + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableSI64(); + items[2147483649L] = 2147483649L; + items[-2147483650L] = -2147483650L; + items[2147483651L] = 2147483651L; + }, + R"_({"SI64":{"2147483649":2147483649,"-2147483650":-2147483650,"2147483651":2147483651}})_"); +} // TestComplexMapAsObject_SI64 Y_UNIT_TEST(TestComplexMapAsObject_FI32) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableFI32(); - items[1073741825U] = 1073741825U; - items[1073741826U] = 1073741826U; - items[1073741827U] = 1073741827U; - }, - R"_({"FI32":{"1073741825":1073741825,"1073741826":1073741826,"1073741827":1073741827}})_"); -} // TestComplexMapAsObject_FI32 + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableFI32(); + items[1073741825U] = 1073741825U; + items[1073741826U] = 1073741826U; + items[1073741827U] = 1073741827U; + }, + R"_({"FI32":{"1073741825":1073741825,"1073741826":1073741826,"1073741827":1073741827}})_"); +} // TestComplexMapAsObject_FI32 Y_UNIT_TEST(TestComplexMapAsObject_FI64) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableFI64(); - items[9223372036854775809UL] = 9223372036854775809UL; - items[9223372036854775810UL] = 9223372036854775810UL; - items[9223372036854775811UL] = 9223372036854775811UL; - }, - R"_({"FI64":{"9223372036854775809":9223372036854775809,"9223372036854775810":9223372036854775810,"9223372036854775811":9223372036854775811}})_"); -} // TestComplexMapAsObject_FI64 + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableFI64(); + items[9223372036854775809UL] = 9223372036854775809UL; + items[9223372036854775810UL] = 9223372036854775810UL; + items[9223372036854775811UL] = 9223372036854775811UL; + }, + R"_({"FI64":{"9223372036854775809":9223372036854775809,"9223372036854775810":9223372036854775810,"9223372036854775811":9223372036854775811}})_"); +} // TestComplexMapAsObject_FI64 Y_UNIT_TEST(TestComplexMapAsObject_SFI32) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableSFI32(); - items[1] = 1; - items[-2] = -2; - items[3] = 3; - }, - R"_({"SFI32":{"1":1,"-2":-2,"3":3}})_"); -} // TestComplexMapAsObject_SFI32 + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableSFI32(); + items[1] = 1; + items[-2] = -2; + items[3] = 3; + }, + R"_({"SFI32":{"1":1,"-2":-2,"3":3}})_"); +} // TestComplexMapAsObject_SFI32 Y_UNIT_TEST(TestComplexMapAsObject_SFI64) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableSFI64(); - items[2147483649L] = 2147483649L; - items[-2147483650L] = -2147483650L; - items[2147483651L] = 2147483651L; - }, - R"_({"SFI64":{"2147483649":2147483649,"-2147483650":-2147483650,"2147483651":2147483651}})_"); -} // TestComplexMapAsObject_SFI64 + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableSFI64(); + items[2147483649L] = 2147483649L; + items[-2147483650L] = -2147483650L; + items[2147483651L] = 2147483651L; + }, + R"_({"SFI64":{"2147483649":2147483649,"-2147483650":-2147483650,"2147483651":2147483651}})_"); +} // TestComplexMapAsObject_SFI64 Y_UNIT_TEST(TestComplexMapAsObject_Bool) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableBool(); - items[true] = true; - items[false] = false; - }, - R"_({"Bool":{"true":true,"false":false}})_"); -} // TestComplexMapAsObject_Bool + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableBool(); + items[true] = true; + items[false] = false; + }, + R"_({"Bool":{"true":true,"false":false}})_"); +} // TestComplexMapAsObject_Bool Y_UNIT_TEST(TestComplexMapAsObject_String) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableString(); - items["key1"] = "value1"; - items["key2"] = "value2"; - items["key3"] = "value3"; + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableString(); + items["key1"] = "value1"; + items["key2"] = "value2"; + items["key3"] = "value3"; items[""] = "value4"; - }, + }, R"_({"String":{"key1":"value1","key2":"value2","key3":"value3","":"value4"}})_"); -} // TestComplexMapAsObject_String +} // TestComplexMapAsObject_String Y_UNIT_TEST(TestComplexMapAsObject_Enum) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableEnum(); - items["key1"] = EEnum::E_1; - items["key2"] = EEnum::E_2; - items["key3"] = EEnum::E_3; - }, - R"_({"Enum":{"key1":1,"key2":2,"key3":3}})_"); -} // TestComplexMapAsObject_Enum + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableEnum(); + items["key1"] = EEnum::E_1; + items["key2"] = EEnum::E_2; + items["key3"] = EEnum::E_3; + }, + R"_({"Enum":{"key1":1,"key2":2,"key3":3}})_"); +} // TestComplexMapAsObject_Enum Y_UNIT_TEST(TestComplexMapAsObject_EnumString) { TestComplexMapAsObject( @@ -944,49 +944,49 @@ Y_UNIT_TEST(TestComplexMapAsObject_EnumStringSnakeCaseInsensitive) { } // TestComplexMapAsObject_EnumStringCaseInsensetive Y_UNIT_TEST(TestComplexMapAsObject_Float) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableFloat(); - items["key1"] = 0.1f; - items["key2"] = 0.2f; - items["key3"] = 0.3f; - }, - R"_({"Float":{"key1":0.1,"key2":0.2,"key3":0.3}})_"); -} // TestComplexMapAsObject_Float + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableFloat(); + items["key1"] = 0.1f; + items["key2"] = 0.2f; + items["key3"] = 0.3f; + }, + R"_({"Float":{"key1":0.1,"key2":0.2,"key3":0.3}})_"); +} // TestComplexMapAsObject_Float Y_UNIT_TEST(TestComplexMapAsObject_Double) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - auto& items = *proto.MutableDouble(); - items["key1"] = 0.1L; - items["key2"] = 0.2L; - items["key3"] = 0.3L; - }, - R"_({"Double":{"key1":0.1,"key2":0.2,"key3":0.3}})_"); -} // TestComplexMapAsObject_Double + TestComplexMapAsObject( + [](TComplexMapType& proto) { + auto& items = *proto.MutableDouble(); + items["key1"] = 0.1L; + items["key2"] = 0.2L; + items["key3"] = 0.3L; + }, + R"_({"Double":{"key1":0.1,"key2":0.2,"key3":0.3}})_"); +} // TestComplexMapAsObject_Double Y_UNIT_TEST(TestComplexMapAsObject_Nested) { - TestComplexMapAsObject( - [](TComplexMapType& proto) { - TComplexMapType inner; - auto& innerItems = *inner.MutableString(); - innerItems["key"] = "value"; - auto& items = *proto.MutableNested(); - items["key1"] = inner; - items["key2"] = inner; - items["key3"] = inner; - }, - R"_({"Nested":{"key1":{"String":{"key":"value"}},"key2":{"String":{"key":"value"}},"key3":{"String":{"key":"value"}}}})_"); -} // TestComplexMapAsObject_Nested + TestComplexMapAsObject( + [](TComplexMapType& proto) { + TComplexMapType inner; + auto& innerItems = *inner.MutableString(); + innerItems["key"] = "value"; + auto& items = *proto.MutableNested(); + items["key1"] = inner; + items["key2"] = inner; + items["key3"] = inner; + }, + R"_({"Nested":{"key1":{"String":{"key":"value"}},"key2":{"String":{"key":"value"}},"key3":{"String":{"key":"value"}}}})_"); +} // TestComplexMapAsObject_Nested Y_UNIT_TEST(TestMapAsObjectConfigNotSet) { - TString modelStr(R"_({"Items":{"key":"value"}})_"); + TString modelStr(R"_({"Items":{"key":"value"}})_"); - TJson2ProtoConfig config; - UNIT_ASSERT_EXCEPTION_CONTAINS( - Json2Proto<TMapType>(modelStr, config), yexception, + TJson2ProtoConfig config; + UNIT_ASSERT_EXCEPTION_CONTAINS( + Json2Proto<TMapType>(modelStr, config), yexception, "Map as object representation is not allowed"); -} // TestMapAsObjectNotSet +} // TestMapAsObjectNotSet Y_UNIT_TEST(TestMergeFlatOptional) { const NJson::TJsonValue& json = CreateFlatJson(); diff --git a/library/cpp/protobuf/json/ut/proto.h b/library/cpp/protobuf/json/ut/proto.h index 8183bfc8e1..b9dec411bd 100644 --- a/library/cpp/protobuf/json/ut/proto.h +++ b/library/cpp/protobuf/json/ut/proto.h @@ -6,57 +6,57 @@ #include <util/system/defaults.h> namespace NProtobufJsonTest { - template <typename TProto> - inline void - FillFlatProto(TProto* proto, - const THashSet<TString>& skippedFields = THashSet<TString>()) { -#define DEFINE_FIELD(name, value) \ - if (skippedFields.find(#name) == skippedFields.end()) \ - proto->Set##name(value); + template <typename TProto> + inline void + FillFlatProto(TProto* proto, + const THashSet<TString>& skippedFields = THashSet<TString>()) { +#define DEFINE_FIELD(name, value) \ + if (skippedFields.find(#name) == skippedFields.end()) \ + proto->Set##name(value); #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD - } - - template <typename TRepeatedField, typename TValue> - inline void - AddValue(TRepeatedField* field, TValue value) { - field->Add(value); - } - - inline void - AddValue(google::protobuf::RepeatedPtrField<TString>* field, const TString& value) { - *(field->Add()) = value; - } - - inline void - FillRepeatedProto(TFlatRepeated* proto, - const THashSet<TString>& skippedFields = THashSet<TString>()) { -#define DEFINE_REPEATED_FIELD(name, type, ...) \ - if (skippedFields.find(#name) == skippedFields.end()) { \ - type values[] = {__VA_ARGS__}; \ - for (size_t i = 0, end = Y_ARRAY_SIZE(values); i < end; ++i) { \ - AddValue(proto->Mutable##name(), values[i]); \ - } \ + } + + template <typename TRepeatedField, typename TValue> + inline void + AddValue(TRepeatedField* field, TValue value) { + field->Add(value); + } + + inline void + AddValue(google::protobuf::RepeatedPtrField<TString>* field, const TString& value) { + *(field->Add()) = value; + } + + inline void + FillRepeatedProto(TFlatRepeated* proto, + const THashSet<TString>& skippedFields = THashSet<TString>()) { +#define DEFINE_REPEATED_FIELD(name, type, ...) \ + if (skippedFields.find(#name) == skippedFields.end()) { \ + type values[] = {__VA_ARGS__}; \ + for (size_t i = 0, end = Y_ARRAY_SIZE(values); i < end; ++i) { \ + AddValue(proto->Mutable##name(), values[i]); \ + } \ } #include <library/cpp/protobuf/json/ut/repeated_fields.incl> #undef DEFINE_REPEATED_FIELD - } - - template <typename TProto> - inline void - FillCompositeProto(TProto* proto, const THashSet<TString>& skippedFields = THashSet<TString>()) { - FillFlatProto(proto->MutablePart(), skippedFields); - } - -#define UNIT_ASSERT_PROTOS_EQUAL(lhs, rhs) \ - do { \ - if (lhs.SerializeAsString() != rhs.SerializeAsString()) { \ - Cerr << ">>>>>>>>>> lhs != rhs:" << Endl; \ - Cerr << lhs.DebugString() << Endl; \ - Cerr << rhs.DebugString() << Endl; \ - UNIT_ASSERT_STRINGS_EQUAL(lhs.DebugString(), rhs.DebugString()); \ + } + + template <typename TProto> + inline void + FillCompositeProto(TProto* proto, const THashSet<TString>& skippedFields = THashSet<TString>()) { + FillFlatProto(proto->MutablePart(), skippedFields); + } + +#define UNIT_ASSERT_PROTOS_EQUAL(lhs, rhs) \ + do { \ + if (lhs.SerializeAsString() != rhs.SerializeAsString()) { \ + Cerr << ">>>>>>>>>> lhs != rhs:" << Endl; \ + Cerr << lhs.DebugString() << Endl; \ + Cerr << rhs.DebugString() << Endl; \ + UNIT_ASSERT_STRINGS_EQUAL(lhs.DebugString(), rhs.DebugString()); \ UNIT_ASSERT_STRINGS_EQUAL(lhs.SerializeAsString(), rhs.SerializeAsString()); \ - } \ + } \ } while (false); -} +} diff --git a/library/cpp/protobuf/json/ut/proto2json_ut.cpp b/library/cpp/protobuf/json/ut/proto2json_ut.cpp index 07e52d7f2f..124ac163cd 100644 --- a/library/cpp/protobuf/json/ut/proto2json_ut.cpp +++ b/library/cpp/protobuf/json/ut/proto2json_ut.cpp @@ -33,43 +33,43 @@ Y_UNIT_TEST_SUITE(TProto2JsonFlatTest) { TProto2JsonConfig cfg; cfg.SetMissingSingleKeyMode(TProto2JsonConfig::MissingKeyDefault); UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, cfg)); -#define DEFINE_FIELD(name, value) \ - { \ - auto descr = proto.GetMetadata().descriptor->FindFieldByName(#name); \ - UNIT_ASSERT(descr); \ - UNIT_ASSERT(json.Has(#name)); \ - switch (descr->cpp_type()) { \ - case FieldDescriptor::CPPTYPE_INT32: \ - UNIT_ASSERT(descr->default_value_int32() == json[#name].GetIntegerRobust()); \ - break; \ - case FieldDescriptor::CPPTYPE_INT64: \ - UNIT_ASSERT(descr->default_value_int64() == json[#name].GetIntegerRobust()); \ - break; \ - case FieldDescriptor::CPPTYPE_UINT32: \ - UNIT_ASSERT(descr->default_value_uint32() == json[#name].GetUIntegerRobust()); \ - break; \ - case FieldDescriptor::CPPTYPE_UINT64: \ - UNIT_ASSERT(descr->default_value_uint32() == json[#name].GetUIntegerRobust()); \ - break; \ - case FieldDescriptor::CPPTYPE_DOUBLE: \ - UNIT_ASSERT(descr->default_value_double() == json[#name].GetDoubleRobust()); \ - break; \ - case FieldDescriptor::CPPTYPE_FLOAT: \ - UNIT_ASSERT(descr->default_value_float() == json[#name].GetDoubleRobust()); \ - break; \ - case FieldDescriptor::CPPTYPE_BOOL: \ - UNIT_ASSERT(descr->default_value_bool() == json[#name].GetBooleanRobust()); \ - break; \ - case FieldDescriptor::CPPTYPE_ENUM: \ - UNIT_ASSERT(descr->default_value_enum()->number() == json[#name].GetIntegerRobust()); \ - break; \ - case FieldDescriptor::CPPTYPE_STRING: \ - UNIT_ASSERT(descr->default_value_string() == json[#name].GetStringRobust()); \ - break; \ - default: \ - break; \ - } \ - } +#define DEFINE_FIELD(name, value) \ + { \ + auto descr = proto.GetMetadata().descriptor->FindFieldByName(#name); \ + UNIT_ASSERT(descr); \ + UNIT_ASSERT(json.Has(#name)); \ + switch (descr->cpp_type()) { \ + case FieldDescriptor::CPPTYPE_INT32: \ + UNIT_ASSERT(descr->default_value_int32() == json[#name].GetIntegerRobust()); \ + break; \ + case FieldDescriptor::CPPTYPE_INT64: \ + UNIT_ASSERT(descr->default_value_int64() == json[#name].GetIntegerRobust()); \ + break; \ + case FieldDescriptor::CPPTYPE_UINT32: \ + UNIT_ASSERT(descr->default_value_uint32() == json[#name].GetUIntegerRobust()); \ + break; \ + case FieldDescriptor::CPPTYPE_UINT64: \ + UNIT_ASSERT(descr->default_value_uint32() == json[#name].GetUIntegerRobust()); \ + break; \ + case FieldDescriptor::CPPTYPE_DOUBLE: \ + UNIT_ASSERT(descr->default_value_double() == json[#name].GetDoubleRobust()); \ + break; \ + case FieldDescriptor::CPPTYPE_FLOAT: \ + UNIT_ASSERT(descr->default_value_float() == json[#name].GetDoubleRobust()); \ + break; \ + case FieldDescriptor::CPPTYPE_BOOL: \ + UNIT_ASSERT(descr->default_value_bool() == json[#name].GetBooleanRobust()); \ + break; \ + case FieldDescriptor::CPPTYPE_ENUM: \ + UNIT_ASSERT(descr->default_value_enum()->number() == json[#name].GetIntegerRobust()); \ + break; \ + case FieldDescriptor::CPPTYPE_STRING: \ + UNIT_ASSERT(descr->default_value_string() == json[#name].GetStringRobust()); \ + break; \ + default: \ + break; \ + } \ + } #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD } @@ -79,7 +79,7 @@ Y_UNIT_TEST_SUITE(TProto2JsonFlatTest) { proto.SetField(42); TProto2JsonConfig cfg; - cfg.SetNameGenerator([](const NProtoBuf::FieldDescriptor&) { return "42"; }); + cfg.SetNameGenerator([](const NProtoBuf::FieldDescriptor&) { return "42"; }); TStringStream str; Proto2Json(proto, str, cfg); @@ -92,7 +92,7 @@ Y_UNIT_TEST_SUITE(TProto2JsonFlatTest) { proto.SetEnum(TEnumValueGeneratorType::ENUM_42); TProto2JsonConfig cfg; - cfg.SetEnumValueGenerator([](const NProtoBuf::EnumValueDescriptor&) { return "42"; }); + cfg.SetEnumValueGenerator([](const NProtoBuf::EnumValueDescriptor&) { return "42"; }); TStringStream str; Proto2Json(proto, str, cfg); @@ -101,370 +101,370 @@ Y_UNIT_TEST_SUITE(TProto2JsonFlatTest) { } Y_UNIT_TEST(TestFlatOptional){ - {TFlatOptional proto; - FillFlatProto(&proto); - const NJson::TJsonValue& modelJson = CreateFlatJson(); - { - NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } - - { - TStringStream jsonStream; - NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); - UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } // streamed -} - - // Try to skip each field -#define DEFINE_FIELD(name, value) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - TFlatOptional proto; \ - FillFlatProto(&proto, skippedField); \ - const NJson::TJsonValue& modelJson = CreateFlatJson(skippedField); \ - NJson::TJsonValue json; \ - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); \ - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ - { \ - TStringStream jsonStream; \ - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); \ - UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); \ - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ - } \ - } + {TFlatOptional proto; + FillFlatProto(&proto); + const NJson::TJsonValue& modelJson = CreateFlatJson(); + { + NJson::TJsonValue json; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } + + { + TStringStream jsonStream; + NJson::TJsonValue json; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); + UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } // streamed +} + + // Try to skip each field +#define DEFINE_FIELD(name, value) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + TFlatOptional proto; \ + FillFlatProto(&proto, skippedField); \ + const NJson::TJsonValue& modelJson = CreateFlatJson(skippedField); \ + NJson::TJsonValue json; \ + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); \ + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ + { \ + TStringStream jsonStream; \ + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); \ + UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); \ + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ + } \ + } #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD -} // TestFlatOptional +} // TestFlatOptional Y_UNIT_TEST(TestFlatRequired){ - {TFlatRequired proto; -FillFlatProto(&proto); -const NJson::TJsonValue& modelJson = CreateFlatJson(); -{ - NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); -} - -{ - TStringStream jsonStream; - NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); - UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); -} // streamed -} - -// Try to skip each field -#define DEFINE_FIELD(name, value) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - TFlatRequired proto; \ - FillFlatProto(&proto, skippedField); \ - const NJson::TJsonValue& modelJson = CreateFlatJson(skippedField); \ - NJson::TJsonValue json; \ - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); \ - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ - { \ - TStringStream jsonStream; \ - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); \ - UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); \ - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ - } \ - } + {TFlatRequired proto; +FillFlatProto(&proto); +const NJson::TJsonValue& modelJson = CreateFlatJson(); +{ + NJson::TJsonValue json; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); +} + +{ + TStringStream jsonStream; + NJson::TJsonValue json; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); + UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); +} // streamed +} + +// Try to skip each field +#define DEFINE_FIELD(name, value) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + TFlatRequired proto; \ + FillFlatProto(&proto, skippedField); \ + const NJson::TJsonValue& modelJson = CreateFlatJson(skippedField); \ + NJson::TJsonValue json; \ + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); \ + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ + { \ + TStringStream jsonStream; \ + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); \ + UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); \ + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ + } \ + } #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD -} // TestFlatRequired +} // TestFlatRequired Y_UNIT_TEST(TestFlatRepeated) { - { - TFlatRepeated proto; - FillRepeatedProto(&proto); - const NJson::TJsonValue& modelJson = CreateRepeatedFlatJson(); + { + TFlatRepeated proto; + FillRepeatedProto(&proto); + const NJson::TJsonValue& modelJson = CreateRepeatedFlatJson(); { - NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + NJson::TJsonValue json; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); } - { - TStringStream jsonStream; - NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); - UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } // streamed - } - - TProto2JsonConfig config; - config.SetMissingRepeatedKeyMode(TProto2JsonConfig::MissingKeySkip); - - // Try to skip each field -#define DEFINE_REPEATED_FIELD(name, ...) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - TFlatRepeated proto; \ - FillRepeatedProto(&proto, skippedField); \ - const NJson::TJsonValue& modelJson = CreateRepeatedFlatJson(skippedField); \ - NJson::TJsonValue json; \ - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); \ - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ - { \ - TStringStream jsonStream; \ - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream, config)); \ - UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); \ - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ - } \ - } + { + TStringStream jsonStream; + NJson::TJsonValue json; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); + UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } // streamed + } + + TProto2JsonConfig config; + config.SetMissingRepeatedKeyMode(TProto2JsonConfig::MissingKeySkip); + + // Try to skip each field +#define DEFINE_REPEATED_FIELD(name, ...) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + TFlatRepeated proto; \ + FillRepeatedProto(&proto, skippedField); \ + const NJson::TJsonValue& modelJson = CreateRepeatedFlatJson(skippedField); \ + NJson::TJsonValue json; \ + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); \ + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ + { \ + TStringStream jsonStream; \ + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream, config)); \ + UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); \ + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ + } \ + } #include <library/cpp/protobuf/json/ut/repeated_fields.incl> #undef DEFINE_REPEATED_FIELD -} // TestFlatRepeated +} // TestFlatRepeated Y_UNIT_TEST(TestCompositeOptional){ - {TCompositeOptional proto; -FillCompositeProto(&proto); -const NJson::TJsonValue& modelJson = CreateCompositeJson(); -{ - NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); -} - -{ - TStringStream jsonStream; - NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); - UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); -} // streamed -} - -// Try to skip each field -#define DEFINE_FIELD(name, value) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - TCompositeOptional proto; \ - FillCompositeProto(&proto, skippedField); \ - const NJson::TJsonValue& modelJson = CreateCompositeJson(skippedField); \ - NJson::TJsonValue json; \ - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); \ - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ - { \ - TStringStream jsonStream; \ - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); \ - UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); \ - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ - } \ - } + {TCompositeOptional proto; +FillCompositeProto(&proto); +const NJson::TJsonValue& modelJson = CreateCompositeJson(); +{ + NJson::TJsonValue json; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); +} + +{ + TStringStream jsonStream; + NJson::TJsonValue json; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); + UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); +} // streamed +} + +// Try to skip each field +#define DEFINE_FIELD(name, value) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + TCompositeOptional proto; \ + FillCompositeProto(&proto, skippedField); \ + const NJson::TJsonValue& modelJson = CreateCompositeJson(skippedField); \ + NJson::TJsonValue json; \ + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); \ + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ + { \ + TStringStream jsonStream; \ + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); \ + UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); \ + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ + } \ + } #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD -} // TestCompositeOptional +} // TestCompositeOptional Y_UNIT_TEST(TestCompositeRequired){ - {TCompositeRequired proto; -FillCompositeProto(&proto); -const NJson::TJsonValue& modelJson = CreateCompositeJson(); -{ - NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); -} - -{ - TStringStream jsonStream; - NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); - UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); -} // streamed -} - -// Try to skip each field -#define DEFINE_FIELD(name, value) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - TCompositeRequired proto; \ - FillCompositeProto(&proto, skippedField); \ - const NJson::TJsonValue& modelJson = CreateCompositeJson(skippedField); \ - NJson::TJsonValue json; \ - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); \ - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ - { \ - TStringStream jsonStream; \ - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); \ - UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); \ - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ - } \ - } + {TCompositeRequired proto; +FillCompositeProto(&proto); +const NJson::TJsonValue& modelJson = CreateCompositeJson(); +{ + NJson::TJsonValue json; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); +} + +{ + TStringStream jsonStream; + NJson::TJsonValue json; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); + UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); +} // streamed +} + +// Try to skip each field +#define DEFINE_FIELD(name, value) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + TCompositeRequired proto; \ + FillCompositeProto(&proto, skippedField); \ + const NJson::TJsonValue& modelJson = CreateCompositeJson(skippedField); \ + NJson::TJsonValue json; \ + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); \ + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ + { \ + TStringStream jsonStream; \ + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); \ + UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); \ + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); \ + } \ + } #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD -} // TestCompositeRequired +} // TestCompositeRequired Y_UNIT_TEST(TestCompositeRepeated) { - { - TFlatOptional partProto; - FillFlatProto(&partProto); - TCompositeRepeated proto; - proto.AddPart()->CopyFrom(partProto); - - NJson::TJsonValue modelJson; - NJson::TJsonValue modelArray; - modelArray.AppendValue(CreateFlatJson()); - modelJson.InsertValue("Part", modelArray); + { + TFlatOptional partProto; + FillFlatProto(&partProto); + TCompositeRepeated proto; + proto.AddPart()->CopyFrom(partProto); + + NJson::TJsonValue modelJson; + NJson::TJsonValue modelArray; + modelArray.AppendValue(CreateFlatJson()); + modelJson.InsertValue("Part", modelArray); { - NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + NJson::TJsonValue json; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); } { - TStringStream jsonStream; - NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); - UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } // streamed - } - - { - // Array of messages with each field skipped - TCompositeRepeated proto; - NJson::TJsonValue modelArray; - -#define DEFINE_REPEATED_FIELD(name, ...) \ - { \ - THashSet<TString> skippedField; \ - skippedField.insert(#name); \ - TFlatOptional partProto; \ - FillFlatProto(&partProto, skippedField); \ - proto.AddPart()->CopyFrom(partProto); \ - modelArray.AppendValue(CreateFlatJson(skippedField)); \ - } + TStringStream jsonStream; + NJson::TJsonValue json; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); + UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } // streamed + } + + { + // Array of messages with each field skipped + TCompositeRepeated proto; + NJson::TJsonValue modelArray; + +#define DEFINE_REPEATED_FIELD(name, ...) \ + { \ + THashSet<TString> skippedField; \ + skippedField.insert(#name); \ + TFlatOptional partProto; \ + FillFlatProto(&partProto, skippedField); \ + proto.AddPart()->CopyFrom(partProto); \ + modelArray.AppendValue(CreateFlatJson(skippedField)); \ + } #include <library/cpp/protobuf/json/ut/repeated_fields.incl> #undef DEFINE_REPEATED_FIELD - NJson::TJsonValue modelJson; - modelJson.InsertValue("Part", modelArray); + NJson::TJsonValue modelJson; + modelJson.InsertValue("Part", modelArray); { NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); UNIT_ASSERT_JSONS_EQUAL(json, modelJson); } { - TStringStream jsonStream; + TStringStream jsonStream; NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); - UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStream)); + UNIT_ASSERT(ReadJsonTree(&jsonStream, &json)); UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } // streamed - } -} // TestCompositeRepeated + } // streamed + } +} // TestCompositeRepeated Y_UNIT_TEST(TestEnumConfig) { - { - TFlatOptional proto; - proto.SetEnum(E_1); - NJson::TJsonValue modelJson; - modelJson.InsertValue("Enum", 1); - NJson::TJsonValue json; - TProto2JsonConfig config; - config.EnumMode = TProto2JsonConfig::EnumNumber; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } - - { - TFlatOptional proto; - proto.SetEnum(E_1); - NJson::TJsonValue modelJson; - modelJson.InsertValue("Enum", "E_1"); - NJson::TJsonValue json; - TProto2JsonConfig config; - config.EnumMode = TProto2JsonConfig::EnumName; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } - - { - TFlatOptional proto; - proto.SetEnum(E_1); - NJson::TJsonValue modelJson; - modelJson.InsertValue("Enum", "NProtobufJsonTest.E_1"); - NJson::TJsonValue json; - TProto2JsonConfig config; - config.EnumMode = TProto2JsonConfig::EnumFullName; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } - - { - TFlatOptional proto; - proto.SetEnum(E_1); - NJson::TJsonValue modelJson; - modelJson.InsertValue("Enum", "e_1"); - NJson::TJsonValue json; - TProto2JsonConfig config; - config.EnumMode = TProto2JsonConfig::EnumNameLowerCase; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } - - { - TFlatOptional proto; - proto.SetEnum(E_1); - NJson::TJsonValue modelJson; - modelJson.InsertValue("Enum", "nprotobufjsontest.e_1"); - NJson::TJsonValue json; - TProto2JsonConfig config; - config.EnumMode = TProto2JsonConfig::EnumFullNameLowerCase; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } -} // TestEnumConfig - + { + TFlatOptional proto; + proto.SetEnum(E_1); + NJson::TJsonValue modelJson; + modelJson.InsertValue("Enum", 1); + NJson::TJsonValue json; + TProto2JsonConfig config; + config.EnumMode = TProto2JsonConfig::EnumNumber; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } + + { + TFlatOptional proto; + proto.SetEnum(E_1); + NJson::TJsonValue modelJson; + modelJson.InsertValue("Enum", "E_1"); + NJson::TJsonValue json; + TProto2JsonConfig config; + config.EnumMode = TProto2JsonConfig::EnumName; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } + + { + TFlatOptional proto; + proto.SetEnum(E_1); + NJson::TJsonValue modelJson; + modelJson.InsertValue("Enum", "NProtobufJsonTest.E_1"); + NJson::TJsonValue json; + TProto2JsonConfig config; + config.EnumMode = TProto2JsonConfig::EnumFullName; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } + + { + TFlatOptional proto; + proto.SetEnum(E_1); + NJson::TJsonValue modelJson; + modelJson.InsertValue("Enum", "e_1"); + NJson::TJsonValue json; + TProto2JsonConfig config; + config.EnumMode = TProto2JsonConfig::EnumNameLowerCase; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } + + { + TFlatOptional proto; + proto.SetEnum(E_1); + NJson::TJsonValue modelJson; + modelJson.InsertValue("Enum", "nprotobufjsontest.e_1"); + NJson::TJsonValue json; + TProto2JsonConfig config; + config.EnumMode = TProto2JsonConfig::EnumFullNameLowerCase; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } +} // TestEnumConfig + Y_UNIT_TEST(TestMissingSingleKeyConfig) { - { - TFlatOptional proto; - NJson::TJsonValue modelJson(NJson::JSON_MAP); - NJson::TJsonValue json; - TProto2JsonConfig config; - config.MissingSingleKeyMode = TProto2JsonConfig::MissingKeySkip; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } - - { - NJson::TJsonValue modelJson; -#define DEFINE_FIELD(name, value) \ - modelJson.InsertValue(#name, NJson::TJsonValue(NJson::JSON_NULL)); + { + TFlatOptional proto; + NJson::TJsonValue modelJson(NJson::JSON_MAP); + NJson::TJsonValue json; + TProto2JsonConfig config; + config.MissingSingleKeyMode = TProto2JsonConfig::MissingKeySkip; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } + + { + NJson::TJsonValue modelJson; +#define DEFINE_FIELD(name, value) \ + modelJson.InsertValue(#name, NJson::TJsonValue(NJson::JSON_NULL)); #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD - TFlatOptional proto; - NJson::TJsonValue json; - TProto2JsonConfig config; - config.MissingSingleKeyMode = TProto2JsonConfig::MissingKeyNull; + TFlatOptional proto; + NJson::TJsonValue json; + TProto2JsonConfig config; + config.MissingSingleKeyMode = TProto2JsonConfig::MissingKeyNull; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } { // Test MissingKeyExplicitDefaultThrowRequired for non explicit default values. TFlatOptional proto; @@ -511,46 +511,46 @@ Y_UNIT_TEST(TestMissingSingleKeyConfig) { UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); UNIT_ASSERT_JSONS_EQUAL(json, modelJson); } -} // TestMissingSingleKeyConfig +} // TestMissingSingleKeyConfig Y_UNIT_TEST(TestMissingRepeatedKeyNoConfig) { - { - TFlatRepeated proto; - NJson::TJsonValue modelJson(NJson::JSON_MAP); - NJson::TJsonValue json; + { + TFlatRepeated proto; + NJson::TJsonValue modelJson(NJson::JSON_MAP); + NJson::TJsonValue json; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } -} // TestMissingRepeatedKeyNoConfig + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } +} // TestMissingRepeatedKeyNoConfig Y_UNIT_TEST(TestMissingRepeatedKeyConfig) { - { - TFlatRepeated proto; - NJson::TJsonValue modelJson(NJson::JSON_MAP); - NJson::TJsonValue json; - TProto2JsonConfig config; - config.MissingRepeatedKeyMode = TProto2JsonConfig::MissingKeySkip; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } - - { - NJson::TJsonValue modelJson; -#define DEFINE_FIELD(name, value) \ - modelJson.InsertValue(#name, NJson::TJsonValue(NJson::JSON_NULL)); + { + TFlatRepeated proto; + NJson::TJsonValue modelJson(NJson::JSON_MAP); + NJson::TJsonValue json; + TProto2JsonConfig config; + config.MissingRepeatedKeyMode = TProto2JsonConfig::MissingKeySkip; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } + + { + NJson::TJsonValue modelJson; +#define DEFINE_FIELD(name, value) \ + modelJson.InsertValue(#name, NJson::TJsonValue(NJson::JSON_NULL)); #include <library/cpp/protobuf/json/ut/fields.incl> #undef DEFINE_FIELD - TFlatRepeated proto; - NJson::TJsonValue json; - TProto2JsonConfig config; - config.MissingRepeatedKeyMode = TProto2JsonConfig::MissingKeyNull; + TFlatRepeated proto; + NJson::TJsonValue json; + TProto2JsonConfig config; + config.MissingRepeatedKeyMode = TProto2JsonConfig::MissingKeyNull; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); - UNIT_ASSERT_JSONS_EQUAL(json, modelJson); - } + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); + UNIT_ASSERT_JSONS_EQUAL(json, modelJson); + } { TFlatRepeated proto; NJson::TJsonValue modelJson(NJson::JSON_MAP); @@ -562,199 +562,199 @@ Y_UNIT_TEST(TestMissingRepeatedKeyConfig) { UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, json, config)); UNIT_ASSERT_JSONS_EQUAL(json, modelJson); } -} // TestMissingRepeatedKeyConfig +} // TestMissingRepeatedKeyConfig Y_UNIT_TEST(TestEscaping) { - // No escape - { - TString modelStr(R"_({"String":"value\""})_"); - - TFlatOptional proto; - proto.SetString(R"_(value")_"); - TStringStream jsonStr; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } - - // TEscapeJTransform - { - TString modelStr(R"_({"String":"value\""})_"); - - TFlatOptional proto; - proto.SetString(R"_(value")_"); - TProto2JsonConfig config; - config.StringTransforms.push_back(new TEscapeJTransform<false, true>()); - TStringStream jsonStr; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(modelStr, jsonStr.Str()); - } - - // TCEscapeTransform - { - TString modelStr(R"_({"String":"value\""})_"); - - TFlatOptional proto; - proto.SetString(R"_(value")_"); - TProto2JsonConfig config; - config.StringTransforms.push_back(new TCEscapeTransform()); - TStringStream jsonStr; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } - - // TSafeUtf8CEscapeTransform - { - TString modelStr(R"_({"String":"value\""})_"); - - TFlatOptional proto; - proto.SetString(R"_(value")_"); - TProto2JsonConfig config; - config.StringTransforms.push_back(new TSafeUtf8CEscapeTransform()); - TStringStream jsonStr; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } -} // TestEscaping - -class TBytesTransform: public IStringTransform { -public: - int GetType() const override { - return 0; - } - void Transform(TString&) const override { - } - void TransformBytes(TString& str) const override { - str = "bytes"; - } -}; + // No escape + { + TString modelStr(R"_({"String":"value\""})_"); + + TFlatOptional proto; + proto.SetString(R"_(value")_"); + TStringStream jsonStr; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } + + // TEscapeJTransform + { + TString modelStr(R"_({"String":"value\""})_"); + + TFlatOptional proto; + proto.SetString(R"_(value")_"); + TProto2JsonConfig config; + config.StringTransforms.push_back(new TEscapeJTransform<false, true>()); + TStringStream jsonStr; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(modelStr, jsonStr.Str()); + } + + // TCEscapeTransform + { + TString modelStr(R"_({"String":"value\""})_"); + + TFlatOptional proto; + proto.SetString(R"_(value")_"); + TProto2JsonConfig config; + config.StringTransforms.push_back(new TCEscapeTransform()); + TStringStream jsonStr; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } + + // TSafeUtf8CEscapeTransform + { + TString modelStr(R"_({"String":"value\""})_"); + + TFlatOptional proto; + proto.SetString(R"_(value")_"); + TProto2JsonConfig config; + config.StringTransforms.push_back(new TSafeUtf8CEscapeTransform()); + TStringStream jsonStr; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } +} // TestEscaping + +class TBytesTransform: public IStringTransform { +public: + int GetType() const override { + return 0; + } + void Transform(TString&) const override { + } + void TransformBytes(TString& str) const override { + str = "bytes"; + } +}; Y_UNIT_TEST(TestBytesTransform) { - // Test that string field is not changed - { - TString modelStr(R"_({"String":"value"})_"); + // Test that string field is not changed + { + TString modelStr(R"_({"String":"value"})_"); - TFlatOptional proto; - proto.SetString(R"_(value)_"); - TProto2JsonConfig config; - config.StringTransforms.push_back(new TBytesTransform()); - TStringStream jsonStr; + TFlatOptional proto; + proto.SetString(R"_(value)_"); + TProto2JsonConfig config; + config.StringTransforms.push_back(new TBytesTransform()); + TStringStream jsonStr; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } - // Test that bytes field is changed - { - TString modelStr(R"_({"Bytes":"bytes"})_"); + // Test that bytes field is changed + { + TString modelStr(R"_({"Bytes":"bytes"})_"); - TFlatOptional proto; - proto.SetBytes(R"_(value)_"); - TProto2JsonConfig config; - config.StringTransforms.push_back(new TBytesTransform()); - TStringStream jsonStr; + TFlatOptional proto; + proto.SetBytes(R"_(value)_"); + TProto2JsonConfig config; + config.StringTransforms.push_back(new TBytesTransform()); + TStringStream jsonStr; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); } -} +} Y_UNIT_TEST(TestFieldNameMode) { - // Original case 1 - { - TString modelStr(R"_({"String":"value"})_"); - - TFlatOptional proto; - proto.SetString("value"); - TStringStream jsonStr; - TProto2JsonConfig config; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } - - // Original case 2 - { - TString modelStr(R"_({"String":"value"})_"); - - TFlatOptional proto; - proto.SetString("value"); - TStringStream jsonStr; - TProto2JsonConfig config; - config.FieldNameMode = TProto2JsonConfig::FieldNameOriginalCase; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } - - // Lowercase - { - TString modelStr(R"_({"string":"value"})_"); - - TFlatOptional proto; - proto.SetString("value"); - TStringStream jsonStr; - TProto2JsonConfig config; - config.FieldNameMode = TProto2JsonConfig::FieldNameLowerCase; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } - - // Uppercase - { - TString modelStr(R"_({"STRING":"value"})_"); - - TFlatOptional proto; - proto.SetString("value"); - TStringStream jsonStr; - TProto2JsonConfig config; - config.FieldNameMode = TProto2JsonConfig::FieldNameUpperCase; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } - - // Camelcase - { - TString modelStr(R"_({"string":"value"})_"); - - TFlatOptional proto; - proto.SetString("value"); - TStringStream jsonStr; - TProto2JsonConfig config; - config.FieldNameMode = TProto2JsonConfig::FieldNameCamelCase; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } - { - TString modelStr(R"_({"oneString":"value"})_"); - - TFlatOptional proto; - proto.SetOneString("value"); - TStringStream jsonStr; - TProto2JsonConfig config; - config.FieldNameMode = TProto2JsonConfig::FieldNameCamelCase; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } - { - TString modelStr(R"_({"oneTwoString":"value"})_"); - - TFlatOptional proto; - proto.SetOneTwoString("value"); - TStringStream jsonStr; - TProto2JsonConfig config; - config.FieldNameMode = TProto2JsonConfig::FieldNameCamelCase; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } + // Original case 1 + { + TString modelStr(R"_({"String":"value"})_"); + + TFlatOptional proto; + proto.SetString("value"); + TStringStream jsonStr; + TProto2JsonConfig config; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } + + // Original case 2 + { + TString modelStr(R"_({"String":"value"})_"); + + TFlatOptional proto; + proto.SetString("value"); + TStringStream jsonStr; + TProto2JsonConfig config; + config.FieldNameMode = TProto2JsonConfig::FieldNameOriginalCase; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } + + // Lowercase + { + TString modelStr(R"_({"string":"value"})_"); + + TFlatOptional proto; + proto.SetString("value"); + TStringStream jsonStr; + TProto2JsonConfig config; + config.FieldNameMode = TProto2JsonConfig::FieldNameLowerCase; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } + + // Uppercase + { + TString modelStr(R"_({"STRING":"value"})_"); + + TFlatOptional proto; + proto.SetString("value"); + TStringStream jsonStr; + TProto2JsonConfig config; + config.FieldNameMode = TProto2JsonConfig::FieldNameUpperCase; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } + + // Camelcase + { + TString modelStr(R"_({"string":"value"})_"); + + TFlatOptional proto; + proto.SetString("value"); + TStringStream jsonStr; + TProto2JsonConfig config; + config.FieldNameMode = TProto2JsonConfig::FieldNameCamelCase; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } + { + TString modelStr(R"_({"oneString":"value"})_"); + + TFlatOptional proto; + proto.SetOneString("value"); + TStringStream jsonStr; + TProto2JsonConfig config; + config.FieldNameMode = TProto2JsonConfig::FieldNameCamelCase; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } + { + TString modelStr(R"_({"oneTwoString":"value"})_"); + + TFlatOptional proto; + proto.SetOneTwoString("value"); + TStringStream jsonStr; + TProto2JsonConfig config; + config.FieldNameMode = TProto2JsonConfig::FieldNameCamelCase; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } // snake_case { @@ -822,189 +822,189 @@ Y_UNIT_TEST(TestFieldNameMode) { UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); } - // Original case, repeated - { - TString modelStr(R"_({"I32":[1,2]})_"); - - TFlatRepeated proto; - proto.AddI32(1); - proto.AddI32(2); - TStringStream jsonStr; - TProto2JsonConfig config; - config.FieldNameMode = TProto2JsonConfig::FieldNameOriginalCase; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } - - // Lower case, repeated - { - TString modelStr(R"_({"i32":[1,2]})_"); - - TFlatRepeated proto; - proto.AddI32(1); - proto.AddI32(2); - TStringStream jsonStr; - TProto2JsonConfig config; - config.FieldNameMode = TProto2JsonConfig::FieldNameLowerCase; - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } - - // UseJsonName - { - // FIXME(CONTRIB-139): see the comment about UseJsonName in json2proto_ut.cpp: - // Def_upper json name should be "DefUpper". - TString modelStr(R"_({"My-Upper":1,"my-lower":2,"defUpper":3,"defLower":4})_"); - - TWithJsonName proto; - proto.Setmy_upper(1); - proto.SetMy_lower(2); - proto.SetDef_upper(3); - proto.Setdef_lower(4); - TStringStream jsonStr; - TProto2JsonConfig config; - config.SetUseJsonName(true); - - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_STRINGS_EQUAL(jsonStr.Str(), modelStr); - } - - // FieldNameMode with UseJsonName - { - TProto2JsonConfig config; - config.SetFieldNameMode(TProto2JsonConfig::FieldNameLowerCase); - UNIT_ASSERT_EXCEPTION_CONTAINS( - config.SetUseJsonName(true), yexception, "mutually exclusive"); - } - { - TProto2JsonConfig config; - config.SetUseJsonName(true); - UNIT_ASSERT_EXCEPTION_CONTAINS( - config.SetFieldNameMode(TProto2JsonConfig::FieldNameLowerCase), yexception, "mutually exclusive"); - } - - /// TODO: test missing keys -} // TestFieldNameMode + // Original case, repeated + { + TString modelStr(R"_({"I32":[1,2]})_"); + + TFlatRepeated proto; + proto.AddI32(1); + proto.AddI32(2); + TStringStream jsonStr; + TProto2JsonConfig config; + config.FieldNameMode = TProto2JsonConfig::FieldNameOriginalCase; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } + + // Lower case, repeated + { + TString modelStr(R"_({"i32":[1,2]})_"); + + TFlatRepeated proto; + proto.AddI32(1); + proto.AddI32(2); + TStringStream jsonStr; + TProto2JsonConfig config; + config.FieldNameMode = TProto2JsonConfig::FieldNameLowerCase; + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } + + // UseJsonName + { + // FIXME(CONTRIB-139): see the comment about UseJsonName in json2proto_ut.cpp: + // Def_upper json name should be "DefUpper". + TString modelStr(R"_({"My-Upper":1,"my-lower":2,"defUpper":3,"defLower":4})_"); + + TWithJsonName proto; + proto.Setmy_upper(1); + proto.SetMy_lower(2); + proto.SetDef_upper(3); + proto.Setdef_lower(4); + TStringStream jsonStr; + TProto2JsonConfig config; + config.SetUseJsonName(true); + + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + UNIT_ASSERT_STRINGS_EQUAL(jsonStr.Str(), modelStr); + } + + // FieldNameMode with UseJsonName + { + TProto2JsonConfig config; + config.SetFieldNameMode(TProto2JsonConfig::FieldNameLowerCase); + UNIT_ASSERT_EXCEPTION_CONTAINS( + config.SetUseJsonName(true), yexception, "mutually exclusive"); + } + { + TProto2JsonConfig config; + config.SetUseJsonName(true); + UNIT_ASSERT_EXCEPTION_CONTAINS( + config.SetFieldNameMode(TProto2JsonConfig::FieldNameLowerCase), yexception, "mutually exclusive"); + } + + /// TODO: test missing keys +} // TestFieldNameMode Y_UNIT_TEST(TestNan) { - TFlatOptional proto; - proto.SetDouble(std::numeric_limits<double>::quiet_NaN()); + TFlatOptional proto; + proto.SetDouble(std::numeric_limits<double>::quiet_NaN()); - UNIT_ASSERT_EXCEPTION(Proto2Json(proto, TProto2JsonConfig()), yexception); -} // TestNan + UNIT_ASSERT_EXCEPTION(Proto2Json(proto, TProto2JsonConfig()), yexception); +} // TestNan Y_UNIT_TEST(TestInf) { - TFlatOptional proto; - proto.SetFloat(std::numeric_limits<float>::infinity()); + TFlatOptional proto; + proto.SetFloat(std::numeric_limits<float>::infinity()); - UNIT_ASSERT_EXCEPTION(Proto2Json(proto, TProto2JsonConfig()), yexception); -} // TestInf + UNIT_ASSERT_EXCEPTION(Proto2Json(proto, TProto2JsonConfig()), yexception); +} // TestInf Y_UNIT_TEST(TestMap) { - TMapType proto; - - auto& items = *proto.MutableItems(); - items["key1"] = "value1"; - items["key2"] = "value2"; - items["key3"] = "value3"; - - TString modelStr(R"_({"Items":[{"key":"key3","value":"value3"},{"key":"key2","value":"value2"},{"key":"key1","value":"value1"}]})_"); - - TStringStream jsonStr; - TProto2JsonConfig config; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - - NJson::TJsonValue jsonValue, modelValue; - NJson::TJsonValue::TArray jsonItems, modelItems; - UNIT_ASSERT(NJson::ReadJsonTree(jsonStr.Str(), &jsonValue)); - UNIT_ASSERT(NJson::ReadJsonTree(modelStr, &modelValue)); - UNIT_ASSERT(jsonValue.Has("Items")); - jsonValue["Items"].GetArray(&jsonItems); - modelValue["Items"].GetArray(&modelItems); - auto itemKey = [](const NJson::TJsonValue& v) { - return v["key"].GetString(); - }; - SortBy(jsonItems, itemKey); - SortBy(modelItems, itemKey); - UNIT_ASSERT_EQUAL(jsonItems, modelItems); -} // TestMap + TMapType proto; + + auto& items = *proto.MutableItems(); + items["key1"] = "value1"; + items["key2"] = "value2"; + items["key3"] = "value3"; + + TString modelStr(R"_({"Items":[{"key":"key3","value":"value3"},{"key":"key2","value":"value2"},{"key":"key1","value":"value1"}]})_"); + + TStringStream jsonStr; + TProto2JsonConfig config; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + + NJson::TJsonValue jsonValue, modelValue; + NJson::TJsonValue::TArray jsonItems, modelItems; + UNIT_ASSERT(NJson::ReadJsonTree(jsonStr.Str(), &jsonValue)); + UNIT_ASSERT(NJson::ReadJsonTree(modelStr, &modelValue)); + UNIT_ASSERT(jsonValue.Has("Items")); + jsonValue["Items"].GetArray(&jsonItems); + modelValue["Items"].GetArray(&modelItems); + auto itemKey = [](const NJson::TJsonValue& v) { + return v["key"].GetString(); + }; + SortBy(jsonItems, itemKey); + SortBy(modelItems, itemKey); + UNIT_ASSERT_EQUAL(jsonItems, modelItems); +} // TestMap Y_UNIT_TEST(TestMapAsObject) { - TMapType proto; + TMapType proto; - auto& items = *proto.MutableItems(); - items["key1"] = "value1"; - items["key2"] = "value2"; - items["key3"] = "value3"; + auto& items = *proto.MutableItems(); + items["key1"] = "value1"; + items["key2"] = "value2"; + items["key3"] = "value3"; - TString modelStr(R"_({"Items":{"key3":"value3","key2":"value2","key1":"value1"}})_"); + TString modelStr(R"_({"Items":{"key3":"value3","key2":"value2","key1":"value1"}})_"); - TStringStream jsonStr; - TProto2JsonConfig config; - config.MapAsObject = true; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); + TStringStream jsonStr; + TProto2JsonConfig config; + config.MapAsObject = true; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); -} // TestMapAsObject + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); +} // TestMapAsObject Y_UNIT_TEST(TestMapWTF) { - TMapType proto; + TMapType proto; - auto& items = *proto.MutableItems(); - items["key1"] = "value1"; - items["key2"] = "value2"; - items["key3"] = "value3"; + auto& items = *proto.MutableItems(); + items["key1"] = "value1"; + items["key2"] = "value2"; + items["key3"] = "value3"; - TString modelStr(R"_({"Items":{"key3":"value3","key2":"value2","key1":"value1"}})_"); + TString modelStr(R"_({"Items":{"key3":"value3","key2":"value2","key1":"value1"}})_"); - TStringStream jsonStr; - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr)); + TStringStream jsonStr; + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr)); - UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); -} // TestMapWTF + UNIT_ASSERT_JSON_STRINGS_EQUAL(jsonStr.Str(), modelStr); +} // TestMapWTF Y_UNIT_TEST(TestStringifyLongNumbers) { -#define TEST_SINGLE(flag, value, expectString) \ - do { \ - TFlatOptional proto; \ - proto.SetSI64(value); \ - \ - TStringStream jsonStr; \ - TProto2JsonConfig config; \ - config.SetStringifyLongNumbers(flag); \ - UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); \ - if (expectString) { \ - UNIT_ASSERT_EQUAL(jsonStr.Str(), "{\"SI64\":\"" #value "\"}"); \ - } else { \ - UNIT_ASSERT_EQUAL(jsonStr.Str(), "{\"SI64\":" #value "}"); \ - } \ - } while (false) - - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersNever, 1, false); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersNever, 1000000000, false); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersNever, 10000000000000000, false); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersNever, -1, false); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersNever, -1000000000, false); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersNever, -10000000000000000, false); - - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForDouble, 1, false); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForDouble, 1000000000, false); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForDouble, 10000000000000000, true); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForDouble, -1, false); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForDouble, -1000000000, false); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForDouble, -10000000000000000, true); - - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForFloat, 1, false); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForFloat, 1000000000, true); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForFloat, 10000000000000000, true); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForFloat, -1, false); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForFloat, -1000000000, true); - TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForFloat, -10000000000000000, true); +#define TEST_SINGLE(flag, value, expectString) \ + do { \ + TFlatOptional proto; \ + proto.SetSI64(value); \ + \ + TStringStream jsonStr; \ + TProto2JsonConfig config; \ + config.SetStringifyLongNumbers(flag); \ + UNIT_ASSERT_NO_EXCEPTION(Proto2Json(proto, jsonStr, config)); \ + if (expectString) { \ + UNIT_ASSERT_EQUAL(jsonStr.Str(), "{\"SI64\":\"" #value "\"}"); \ + } else { \ + UNIT_ASSERT_EQUAL(jsonStr.Str(), "{\"SI64\":" #value "}"); \ + } \ + } while (false) + + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersNever, 1, false); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersNever, 1000000000, false); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersNever, 10000000000000000, false); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersNever, -1, false); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersNever, -1000000000, false); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersNever, -10000000000000000, false); + + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForDouble, 1, false); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForDouble, 1000000000, false); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForDouble, 10000000000000000, true); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForDouble, -1, false); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForDouble, -1000000000, false); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForDouble, -10000000000000000, true); + + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForFloat, 1, false); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForFloat, 1000000000, true); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForFloat, 10000000000000000, true); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForFloat, -1, false); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForFloat, -1000000000, true); + TEST_SINGLE(TProto2JsonConfig::StringifyLongNumbersForFloat, -10000000000000000, true); #undef TEST_SINGLE -} // TestStringifyLongNumbers +} // TestStringifyLongNumbers Y_UNIT_TEST(TestExtension) { TExtensionField proto; diff --git a/library/cpp/protobuf/json/ut/string_transform_ut.cpp b/library/cpp/protobuf/json/ut/string_transform_ut.cpp index a31dabcb0f..90debecd66 100644 --- a/library/cpp/protobuf/json/ut/string_transform_ut.cpp +++ b/library/cpp/protobuf/json/ut/string_transform_ut.cpp @@ -40,7 +40,7 @@ Y_UNIT_TEST_SUITE(TDoubleEscapeTransform) { Y_UNIT_TEST_SUITE(TDoubleUnescapeTransform) { Y_UNIT_TEST(TestEmptyString) { - const NProtobufJson::IStringTransform& transform = NProtobufJson::TDoubleUnescapeTransform(); + const NProtobufJson::IStringTransform& transform = NProtobufJson::TDoubleUnescapeTransform(); TString s; s = ""; transform.Transform(s); @@ -48,7 +48,7 @@ Y_UNIT_TEST_SUITE(TDoubleUnescapeTransform) { } Y_UNIT_TEST(TestAlphabeticString) { - const NProtobufJson::IStringTransform& transform = NProtobufJson::TDoubleUnescapeTransform(); + const NProtobufJson::IStringTransform& transform = NProtobufJson::TDoubleUnescapeTransform(); TString s; s = "abacaba"; transform.Transform(s); @@ -57,7 +57,7 @@ Y_UNIT_TEST_SUITE(TDoubleUnescapeTransform) { } Y_UNIT_TEST(TestRussianSymbols) { - const NProtobufJson::IStringTransform& transform = NProtobufJson::TDoubleUnescapeTransform(); + const NProtobufJson::IStringTransform& transform = NProtobufJson::TDoubleUnescapeTransform(); TString s; s = "\\\\321\\\\202\\\\320\\\\265\\\\321\\\\201\\\\321\\\\202"; transform.Transform(s); @@ -65,7 +65,7 @@ Y_UNIT_TEST_SUITE(TDoubleUnescapeTransform) { } Y_UNIT_TEST(TestEscapeSpecialSymbols) { - const NProtobufJson::IStringTransform& transform = NProtobufJson::TDoubleUnescapeTransform(); + const NProtobufJson::IStringTransform& transform = NProtobufJson::TDoubleUnescapeTransform(); TString s; s = "aba\\\\\\\\ca\\\\\\\"ba"; transform.Transform(s); @@ -73,7 +73,7 @@ Y_UNIT_TEST_SUITE(TDoubleUnescapeTransform) { } Y_UNIT_TEST(TestEscapeSpecialSymbolsDifficultCases) { - const NProtobufJson::IStringTransform& transform = NProtobufJson::TDoubleUnescapeTransform(); + const NProtobufJson::IStringTransform& transform = NProtobufJson::TDoubleUnescapeTransform(); TString s; s = "\\\\\\\\\\\\\\\\"; transform.Transform(s); diff --git a/library/cpp/protobuf/util/cast.h b/library/cpp/protobuf/util/cast.h index 83749dfcee..e99d4f200a 100644 --- a/library/cpp/protobuf/util/cast.h +++ b/library/cpp/protobuf/util/cast.h @@ -1,5 +1,5 @@ #pragma once - + #include "traits.h" #include <google/protobuf/descriptor.h> @@ -8,149 +8,149 @@ #include <util/generic/cast.h> namespace NProtoBuf { - // C++ compatible conversions of FieldDescriptor::CppType's - - using ECppType = FieldDescriptor::CppType; + // C++ compatible conversions of FieldDescriptor::CppType's - namespace NCast { - template <ECppType src, ECppType dst> - struct TIsCompatibleCppType { - enum { - Result = src == dst || - (TIsNumericCppType<src>::Result && TIsNumericCppType<dst>::Result) - }; - }; + using ECppType = FieldDescriptor::CppType; - template <ECppType src, ECppType dst> - struct TIsEnumToNumericCppType { - enum { - Result = (src == FieldDescriptor::CPPTYPE_ENUM && TIsNumericCppType<dst>::Result) - }; + namespace NCast { + template <ECppType src, ECppType dst> + struct TIsCompatibleCppType { + enum { + Result = src == dst || + (TIsNumericCppType<src>::Result && TIsNumericCppType<dst>::Result) + }; }; - template <ECppType src, ECppType dst, bool compatible> // compatible == true - struct TCompatCastBase { - static const bool IsCompatible = true; - - typedef typename TCppTypeTraits<src>::T TSrc; - typedef typename TCppTypeTraits<dst>::T TDst; - - static inline TDst Cast(TSrc value) { - return value; - } + template <ECppType src, ECppType dst> + struct TIsEnumToNumericCppType { + enum { + Result = (src == FieldDescriptor::CPPTYPE_ENUM && TIsNumericCppType<dst>::Result) + }; }; - template <ECppType src, ECppType dst> // compatible == false - struct TCompatCastBase<src, dst, false> { - static const bool IsCompatible = false; - - typedef typename TCppTypeTraits<src>::T TSrc; - typedef typename TCppTypeTraits<dst>::T TDst; - - static inline TDst Cast(TSrc) { - ythrow TBadCastException() << "Incompatible FieldDescriptor::CppType conversion: #" - << (size_t)src << " to #" << (size_t)dst; - } - }; - - template <ECppType src, ECppType dst, bool isEnumToNum> // enum -> numeric - struct TCompatCastImpl { - static const bool IsCompatible = true; - - typedef typename TCppTypeTraits<dst>::T TDst; - - static inline TDst Cast(const EnumValueDescriptor* value) { - Y_ASSERT(value != nullptr); - return value->number(); - } - }; - - template <ECppType src, ECppType dst> - struct TCompatCastImpl<src, dst, false>: public TCompatCastBase<src, dst, TIsCompatibleCppType<src, dst>::Result> { - using TCompatCastBase<src, dst, TIsCompatibleCppType<src, dst>::Result>::IsCompatible; - }; - - template <ECppType src, ECppType dst> - struct TCompatCast: public TCompatCastImpl<src, dst, TIsEnumToNumericCppType<src, dst>::Result> { - typedef TCompatCastImpl<src, dst, TIsEnumToNumericCppType<src, dst>::Result> TBase; - - typedef typename TCppTypeTraits<src>::T TSrc; - typedef typename TCppTypeTraits<dst>::T TDst; - - using TBase::Cast; - using TBase::IsCompatible; - - inline bool Try(TSrc value, TDst& res) { - if (IsCompatible) { - res = Cast(value); - return true; - } - return false; - } - }; - - } - + template <ECppType src, ECppType dst, bool compatible> // compatible == true + struct TCompatCastBase { + static const bool IsCompatible = true; + + typedef typename TCppTypeTraits<src>::T TSrc; + typedef typename TCppTypeTraits<dst>::T TDst; + + static inline TDst Cast(TSrc value) { + return value; + } + }; + + template <ECppType src, ECppType dst> // compatible == false + struct TCompatCastBase<src, dst, false> { + static const bool IsCompatible = false; + + typedef typename TCppTypeTraits<src>::T TSrc; + typedef typename TCppTypeTraits<dst>::T TDst; + + static inline TDst Cast(TSrc) { + ythrow TBadCastException() << "Incompatible FieldDescriptor::CppType conversion: #" + << (size_t)src << " to #" << (size_t)dst; + } + }; + + template <ECppType src, ECppType dst, bool isEnumToNum> // enum -> numeric + struct TCompatCastImpl { + static const bool IsCompatible = true; + + typedef typename TCppTypeTraits<dst>::T TDst; + + static inline TDst Cast(const EnumValueDescriptor* value) { + Y_ASSERT(value != nullptr); + return value->number(); + } + }; + + template <ECppType src, ECppType dst> + struct TCompatCastImpl<src, dst, false>: public TCompatCastBase<src, dst, TIsCompatibleCppType<src, dst>::Result> { + using TCompatCastBase<src, dst, TIsCompatibleCppType<src, dst>::Result>::IsCompatible; + }; + + template <ECppType src, ECppType dst> + struct TCompatCast: public TCompatCastImpl<src, dst, TIsEnumToNumericCppType<src, dst>::Result> { + typedef TCompatCastImpl<src, dst, TIsEnumToNumericCppType<src, dst>::Result> TBase; + + typedef typename TCppTypeTraits<src>::T TSrc; + typedef typename TCppTypeTraits<dst>::T TDst; + + using TBase::Cast; + using TBase::IsCompatible; + + inline bool Try(TSrc value, TDst& res) { + if (IsCompatible) { + res = Cast(value); + return true; + } + return false; + } + }; + + } + template <ECppType src, ECppType dst> - inline typename TCppTypeTraits<dst>::T CompatCast(typename TCppTypeTraits<src>::T value) { - return NCast::TCompatCast<src, dst>::Cast(value); - } + inline typename TCppTypeTraits<dst>::T CompatCast(typename TCppTypeTraits<src>::T value) { + return NCast::TCompatCast<src, dst>::Cast(value); + } template <ECppType src, ECppType dst> - inline bool TryCompatCast(typename TCppTypeTraits<src>::T value, typename TCppTypeTraits<dst>::T& res) { - return NCast::TCompatCast<src, dst>::Try(value, res); - } - - // Message static/dynamic checked casts - - template <typename TpMessage> - inline const TpMessage* TryCast(const Message* msg) { - if (!msg || TpMessage::descriptor() != msg->GetDescriptor()) - return NULL; - return CheckedCast<const TpMessage*>(msg); - } - - template <typename TpMessage> - inline const TpMessage* TryCast(const Message* msg, const TpMessage*& ret) { - ret = TryCast<TpMessage>(msg); - return ret; - } - - template <typename TpMessage> - inline TpMessage* TryCast(Message* msg) { - if (!msg || TpMessage::descriptor() != msg->GetDescriptor()) - return nullptr; - return CheckedCast<TpMessage*>(msg); - } - - template <typename TpMessage> - inline TpMessage* TryCast(Message* msg, TpMessage*& ret) { - ret = TryCast<TpMessage>(msg); - return ret; - } - - // specialize for Message itself - - template <> - inline const Message* TryCast<Message>(const Message* msg) { - return msg; - } - - template <> - inline Message* TryCast<Message>(Message* msg) { - return msg; - } - - // Binary serialization compatible conversion - inline bool TryBinaryCast(const Message* from, Message* to, TString* buffer = nullptr) { - TString tmpbuf; - if (!buffer) - buffer = &tmpbuf; - - if (!from->SerializeToString(buffer)) - return false; - - return to->ParseFromString(*buffer); - } + inline bool TryCompatCast(typename TCppTypeTraits<src>::T value, typename TCppTypeTraits<dst>::T& res) { + return NCast::TCompatCast<src, dst>::Try(value, res); + } + + // Message static/dynamic checked casts + + template <typename TpMessage> + inline const TpMessage* TryCast(const Message* msg) { + if (!msg || TpMessage::descriptor() != msg->GetDescriptor()) + return NULL; + return CheckedCast<const TpMessage*>(msg); + } + + template <typename TpMessage> + inline const TpMessage* TryCast(const Message* msg, const TpMessage*& ret) { + ret = TryCast<TpMessage>(msg); + return ret; + } + + template <typename TpMessage> + inline TpMessage* TryCast(Message* msg) { + if (!msg || TpMessage::descriptor() != msg->GetDescriptor()) + return nullptr; + return CheckedCast<TpMessage*>(msg); + } + + template <typename TpMessage> + inline TpMessage* TryCast(Message* msg, TpMessage*& ret) { + ret = TryCast<TpMessage>(msg); + return ret; + } + + // specialize for Message itself + + template <> + inline const Message* TryCast<Message>(const Message* msg) { + return msg; + } + + template <> + inline Message* TryCast<Message>(Message* msg) { + return msg; + } + + // Binary serialization compatible conversion + inline bool TryBinaryCast(const Message* from, Message* to, TString* buffer = nullptr) { + TString tmpbuf; + if (!buffer) + buffer = &tmpbuf; + + if (!from->SerializeToString(buffer)) + return false; + + return to->ParseFromString(*buffer); + } } diff --git a/library/cpp/protobuf/util/is_equal.cpp b/library/cpp/protobuf/util/is_equal.cpp index 227408006e..f9711c9565 100644 --- a/library/cpp/protobuf/util/is_equal.cpp +++ b/library/cpp/protobuf/util/is_equal.cpp @@ -8,156 +8,156 @@ #include <util/string/vector.h> namespace NProtoBuf { - template <bool useDefault> - static bool IsEqualImpl(const Message& m1, const Message& m2, TVector<TString>* differentPath); - - namespace { - template <FieldDescriptor::CppType CppType, bool useDefault> - struct TCompareValue { - typedef typename TCppTypeTraits<CppType>::T T; - static inline bool IsEqual(T value1, T value2, TVector<TString>*) { - return value1 == value2; - } - }; - - template <bool useDefault> - struct TCompareValue<FieldDescriptor::CPPTYPE_MESSAGE, useDefault> { - static inline bool IsEqual(const Message* value1, const Message* value2, TVector<TString>* differentPath) { - return NProtoBuf::IsEqualImpl<useDefault>(*value1, *value2, differentPath); - } - }; - - template <FieldDescriptor::CppType CppType, bool useDefault> - class TCompareField { - typedef TCppTypeTraits<CppType> TTraits; - typedef TCompareValue<CppType, useDefault> TCompare; - - public: - static inline bool IsEqual(const Message& m1, const Message& m2, const FieldDescriptor& field, TVector<TString>* differentPath) { - if (field.is_repeated()) - return IsEqualRepeated(m1, m2, &field, differentPath); - else - return IsEqualSingle(m1, m2, &field, differentPath); - } - - private: - static bool IsEqualSingle(const Message& m1, const Message& m2, const FieldDescriptor* field, TVector<TString>* differentPath) { - bool has1 = m1.GetReflection()->HasField(m1, field); - bool has2 = m2.GetReflection()->HasField(m2, field); - - if (has1 != has2) { - if (!useDefault || field->is_required()) { - return false; - } - } else if (!has1) - return true; - - return TCompare::IsEqual(TTraits::Get(m1, field), - TTraits::Get(m2, field), - differentPath); - } - - static bool IsEqualRepeated(const Message& m1, const Message& m2, const FieldDescriptor* field, TVector<TString>* differentPath) { - int fieldSize = m1.GetReflection()->FieldSize(m1, field); - if (fieldSize != m2.GetReflection()->FieldSize(m2, field)) + template <bool useDefault> + static bool IsEqualImpl(const Message& m1, const Message& m2, TVector<TString>* differentPath); + + namespace { + template <FieldDescriptor::CppType CppType, bool useDefault> + struct TCompareValue { + typedef typename TCppTypeTraits<CppType>::T T; + static inline bool IsEqual(T value1, T value2, TVector<TString>*) { + return value1 == value2; + } + }; + + template <bool useDefault> + struct TCompareValue<FieldDescriptor::CPPTYPE_MESSAGE, useDefault> { + static inline bool IsEqual(const Message* value1, const Message* value2, TVector<TString>* differentPath) { + return NProtoBuf::IsEqualImpl<useDefault>(*value1, *value2, differentPath); + } + }; + + template <FieldDescriptor::CppType CppType, bool useDefault> + class TCompareField { + typedef TCppTypeTraits<CppType> TTraits; + typedef TCompareValue<CppType, useDefault> TCompare; + + public: + static inline bool IsEqual(const Message& m1, const Message& m2, const FieldDescriptor& field, TVector<TString>* differentPath) { + if (field.is_repeated()) + return IsEqualRepeated(m1, m2, &field, differentPath); + else + return IsEqualSingle(m1, m2, &field, differentPath); + } + + private: + static bool IsEqualSingle(const Message& m1, const Message& m2, const FieldDescriptor* field, TVector<TString>* differentPath) { + bool has1 = m1.GetReflection()->HasField(m1, field); + bool has2 = m2.GetReflection()->HasField(m2, field); + + if (has1 != has2) { + if (!useDefault || field->is_required()) { + return false; + } + } else if (!has1) + return true; + + return TCompare::IsEqual(TTraits::Get(m1, field), + TTraits::Get(m2, field), + differentPath); + } + + static bool IsEqualRepeated(const Message& m1, const Message& m2, const FieldDescriptor* field, TVector<TString>* differentPath) { + int fieldSize = m1.GetReflection()->FieldSize(m1, field); + if (fieldSize != m2.GetReflection()->FieldSize(m2, field)) return false; - for (int i = 0; i < fieldSize; ++i) - if (!IsEqualRepeatedValue(m1, m2, field, i, differentPath)) { - if (!!differentPath) { - differentPath->push_back(ToString(i)); - } - return false; - } + for (int i = 0; i < fieldSize; ++i) + if (!IsEqualRepeatedValue(m1, m2, field, i, differentPath)) { + if (!!differentPath) { + differentPath->push_back(ToString(i)); + } + return false; + } return true; - } - - static inline bool IsEqualRepeatedValue(const Message& m1, const Message& m2, const FieldDescriptor* field, int index, TVector<TString>* differentPath) { - return TCompare::IsEqual(TTraits::GetRepeated(m1, field, index), - TTraits::GetRepeated(m2, field, index), - differentPath); - } - }; - - template <bool useDefault> - bool IsEqualField(const Message& m1, const Message& m2, const FieldDescriptor& field, TVector<TString>* differentPath) { -#define CASE_CPPTYPE(cpptype) \ - case FieldDescriptor::CPPTYPE_##cpptype: { \ - bool r = TCompareField<FieldDescriptor::CPPTYPE_##cpptype, useDefault>::IsEqual(m1, m2, field, differentPath); \ - if (!r && !!differentPath) { \ - differentPath->push_back(field.name()); \ - } \ - return r; \ - } - - switch (field.cpp_type()) { - CASE_CPPTYPE(INT32) - CASE_CPPTYPE(INT64) - CASE_CPPTYPE(UINT32) - CASE_CPPTYPE(UINT64) - CASE_CPPTYPE(DOUBLE) - CASE_CPPTYPE(FLOAT) - CASE_CPPTYPE(BOOL) - CASE_CPPTYPE(ENUM) - CASE_CPPTYPE(STRING) - CASE_CPPTYPE(MESSAGE) - default: - ythrow yexception() << "Unsupported cpp-type field comparison"; - } - -#undef CASE_CPPTYPE + } + + static inline bool IsEqualRepeatedValue(const Message& m1, const Message& m2, const FieldDescriptor* field, int index, TVector<TString>* differentPath) { + return TCompare::IsEqual(TTraits::GetRepeated(m1, field, index), + TTraits::GetRepeated(m2, field, index), + differentPath); + } + }; + + template <bool useDefault> + bool IsEqualField(const Message& m1, const Message& m2, const FieldDescriptor& field, TVector<TString>* differentPath) { +#define CASE_CPPTYPE(cpptype) \ + case FieldDescriptor::CPPTYPE_##cpptype: { \ + bool r = TCompareField<FieldDescriptor::CPPTYPE_##cpptype, useDefault>::IsEqual(m1, m2, field, differentPath); \ + if (!r && !!differentPath) { \ + differentPath->push_back(field.name()); \ + } \ + return r; \ + } + + switch (field.cpp_type()) { + CASE_CPPTYPE(INT32) + CASE_CPPTYPE(INT64) + CASE_CPPTYPE(UINT32) + CASE_CPPTYPE(UINT64) + CASE_CPPTYPE(DOUBLE) + CASE_CPPTYPE(FLOAT) + CASE_CPPTYPE(BOOL) + CASE_CPPTYPE(ENUM) + CASE_CPPTYPE(STRING) + CASE_CPPTYPE(MESSAGE) + default: + ythrow yexception() << "Unsupported cpp-type field comparison"; + } + +#undef CASE_CPPTYPE } - } + } template <bool useDefault> - bool IsEqualImpl(const Message& m1, const Message& m2, TVector<TString>* differentPath) { - const Descriptor* descr = m1.GetDescriptor(); - if (descr != m2.GetDescriptor()) { - return false; + bool IsEqualImpl(const Message& m1, const Message& m2, TVector<TString>* differentPath) { + const Descriptor* descr = m1.GetDescriptor(); + if (descr != m2.GetDescriptor()) { + return false; } - for (int i = 0; i < descr->field_count(); ++i) - if (!IsEqualField<useDefault>(m1, m2, *descr->field(i), differentPath)) { - return false; - } - return true; - } - - bool IsEqual(const Message& m1, const Message& m2) { - return IsEqualImpl<false>(m1, m2, nullptr); - } - - bool IsEqual(const Message& m1, const Message& m2, TString* differentPath) { - TVector<TString> differentPathVector; - TVector<TString>* differentPathVectorPtr = !!differentPath ? &differentPathVector : nullptr; - bool r = IsEqualImpl<false>(m1, m2, differentPathVectorPtr); - if (!r && differentPath) { - *differentPath = JoinStrings(differentPathVector.rbegin(), differentPathVector.rend(), "/"); + for (int i = 0; i < descr->field_count(); ++i) + if (!IsEqualField<useDefault>(m1, m2, *descr->field(i), differentPath)) { + return false; + } + return true; + } + + bool IsEqual(const Message& m1, const Message& m2) { + return IsEqualImpl<false>(m1, m2, nullptr); + } + + bool IsEqual(const Message& m1, const Message& m2, TString* differentPath) { + TVector<TString> differentPathVector; + TVector<TString>* differentPathVectorPtr = !!differentPath ? &differentPathVector : nullptr; + bool r = IsEqualImpl<false>(m1, m2, differentPathVectorPtr); + if (!r && differentPath) { + *differentPath = JoinStrings(differentPathVector.rbegin(), differentPathVector.rend(), "/"); } - return r; - } + return r; + } - bool IsEqualDefault(const Message& m1, const Message& m2) { - return IsEqualImpl<true>(m1, m2, nullptr); + bool IsEqualDefault(const Message& m1, const Message& m2) { + return IsEqualImpl<true>(m1, m2, nullptr); } - template <bool useDefault> - static bool IsEqualFieldImpl( - const Message& m1, - const Message& m2, - const FieldDescriptor& field, - TVector<TString>* differentPath) { - const Descriptor* descr = m1.GetDescriptor(); - if (descr != m2.GetDescriptor()) { + template <bool useDefault> + static bool IsEqualFieldImpl( + const Message& m1, + const Message& m2, + const FieldDescriptor& field, + TVector<TString>* differentPath) { + const Descriptor* descr = m1.GetDescriptor(); + if (descr != m2.GetDescriptor()) { return false; } - return IsEqualField<useDefault>(m1, m2, field, differentPath); - } + return IsEqualField<useDefault>(m1, m2, field, differentPath); + } - bool IsEqualField(const Message& m1, const Message& m2, const FieldDescriptor& field) { - return IsEqualFieldImpl<false>(m1, m2, field, nullptr); + bool IsEqualField(const Message& m1, const Message& m2, const FieldDescriptor& field) { + return IsEqualFieldImpl<false>(m1, m2, field, nullptr); } - bool IsEqualFieldDefault(const Message& m1, const Message& m2, const FieldDescriptor& field) { - return IsEqualFieldImpl<true>(m1, m2, field, nullptr); + bool IsEqualFieldDefault(const Message& m1, const Message& m2, const FieldDescriptor& field) { + return IsEqualFieldImpl<true>(m1, m2, field, nullptr); } } diff --git a/library/cpp/protobuf/util/is_equal.h b/library/cpp/protobuf/util/is_equal.h index 13c0aae63d..46d38591ae 100644 --- a/library/cpp/protobuf/util/is_equal.h +++ b/library/cpp/protobuf/util/is_equal.h @@ -10,24 +10,24 @@ namespace google { } namespace NProtoBuf { - using ::google::protobuf::FieldDescriptor; + using ::google::protobuf::FieldDescriptor; using ::google::protobuf::Message; } namespace NProtoBuf { - // Reflection-based equality check for arbitrary protobuf messages + // Reflection-based equality check for arbitrary protobuf messages - // Strict comparison: optional field without value is NOT equal to - // a field with explicitly set default value. - bool IsEqual(const Message& m1, const Message& m2); - bool IsEqual(const Message& m1, const Message& m2, TString* differentPath); + // Strict comparison: optional field without value is NOT equal to + // a field with explicitly set default value. + bool IsEqual(const Message& m1, const Message& m2); + bool IsEqual(const Message& m1, const Message& m2, TString* differentPath); - bool IsEqualField(const Message& m1, const Message& m2, const FieldDescriptor& field); + bool IsEqualField(const Message& m1, const Message& m2, const FieldDescriptor& field); - // Non-strict version: optional field without explicit value is compared - // using its default value. - bool IsEqualDefault(const Message& m1, const Message& m2); + // Non-strict version: optional field without explicit value is compared + // using its default value. + bool IsEqualDefault(const Message& m1, const Message& m2); - bool IsEqualFieldDefault(const Message& m1, const Message& m2, const FieldDescriptor& field); + bool IsEqualFieldDefault(const Message& m1, const Message& m2, const FieldDescriptor& field); } diff --git a/library/cpp/protobuf/util/merge.cpp b/library/cpp/protobuf/util/merge.cpp index dc2b9cc806..258cbb737f 100644 --- a/library/cpp/protobuf/util/merge.cpp +++ b/library/cpp/protobuf/util/merge.cpp @@ -6,41 +6,41 @@ #include <library/cpp/protobuf/util/proto/merge.pb.h> namespace NProtoBuf { - void RewriteMerge(const Message& src, Message& dst) { - const Descriptor* d = src.GetDescriptor(); - Y_ASSERT(d == dst.GetDescriptor()); + void RewriteMerge(const Message& src, Message& dst) { + const Descriptor* d = src.GetDescriptor(); + Y_ASSERT(d == dst.GetDescriptor()); - for (int i = 0; i < d->field_count(); ++i) { - if (TConstField(src, d->field(i)).Has()) - TMutableField(dst, d->field(i)).Clear(); - } + for (int i = 0; i < d->field_count(); ++i) { + if (TConstField(src, d->field(i)).Has()) + TMutableField(dst, d->field(i)).Clear(); + } - dst.MergeFrom(src); + dst.MergeFrom(src); } - static void ClearNonMergeable(const Message& src, Message& dst) { - const Descriptor* d = src.GetDescriptor(); - if (d->options().GetExtension(DontMerge)) { - dst.Clear(); - return; - } - - for (int i = 0; i < d->field_count(); ++i) { - const FieldDescriptor* fd = d->field(i); - TConstField srcField(src, fd); - if (srcField.Has()) { - TMutableField dstField(dst, fd); - if (fd->options().GetExtension(DontMergeField)) - dstField.Clear(); - else if (!fd->is_repeated() && dstField.IsMessage() && dstField.Has()) - ClearNonMergeable(*srcField.Get<const Message*>(), *dstField.MutableMessage()); - } - } + static void ClearNonMergeable(const Message& src, Message& dst) { + const Descriptor* d = src.GetDescriptor(); + if (d->options().GetExtension(DontMerge)) { + dst.Clear(); + return; + } + + for (int i = 0; i < d->field_count(); ++i) { + const FieldDescriptor* fd = d->field(i); + TConstField srcField(src, fd); + if (srcField.Has()) { + TMutableField dstField(dst, fd); + if (fd->options().GetExtension(DontMergeField)) + dstField.Clear(); + else if (!fd->is_repeated() && dstField.IsMessage() && dstField.Has()) + ClearNonMergeable(*srcField.Get<const Message*>(), *dstField.MutableMessage()); + } + } } - void CustomMerge(const Message& src, Message& dst) { - ClearNonMergeable(src, dst); - dst.MergeFrom(src); + void CustomMerge(const Message& src, Message& dst) { + ClearNonMergeable(src, dst); + dst.MergeFrom(src); } } diff --git a/library/cpp/protobuf/util/merge.h b/library/cpp/protobuf/util/merge.h index 924975f141..b354331788 100644 --- a/library/cpp/protobuf/util/merge.h +++ b/library/cpp/protobuf/util/merge.h @@ -11,12 +11,12 @@ namespace NProtoBuf { } namespace NProtoBuf { - // Similiar to Message::MergeFrom, overwrites existing repeated fields - // and embedded messages completely instead of recursive merging. - void RewriteMerge(const Message& src, Message& dst); + // Similiar to Message::MergeFrom, overwrites existing repeated fields + // and embedded messages completely instead of recursive merging. + void RewriteMerge(const Message& src, Message& dst); - // Does standard MergeFrom() by default, except messages/fields marked with DontMerge or DontMergeField option. - // Such fields are merged using RewriteMerge() (i.e. destination is cleared before merging anything from source) - void CustomMerge(const Message& src, Message& dst); + // Does standard MergeFrom() by default, except messages/fields marked with DontMerge or DontMergeField option. + // Such fields are merged using RewriteMerge() (i.e. destination is cleared before merging anything from source) + void CustomMerge(const Message& src, Message& dst); -} +} diff --git a/library/cpp/protobuf/util/merge_ut.cpp b/library/cpp/protobuf/util/merge_ut.cpp index 22217db183..84b40c24b8 100644 --- a/library/cpp/protobuf/util/merge_ut.cpp +++ b/library/cpp/protobuf/util/merge_ut.cpp @@ -6,7 +6,7 @@ using namespace NProtoBuf; Y_UNIT_TEST_SUITE(ProtobufMerge) { - static void InitProto(NProtobufUtilUt::TMergeTest & p, bool isSrc) { + static void InitProto(NProtobufUtilUt::TMergeTest & p, bool isSrc) { size_t start = isSrc ? 0 : 100; p.AddMergeInt(start + 1); @@ -40,10 +40,10 @@ Y_UNIT_TEST_SUITE(ProtobufMerge) { InitProto(src, true); InitProto(dst, false); - // Cerr << "\nsrc: " << src.ShortDebugString() << Endl; - // Cerr << "dst: " << dst.ShortDebugString() << Endl; + // Cerr << "\nsrc: " << src.ShortDebugString() << Endl; + // Cerr << "dst: " << dst.ShortDebugString() << Endl; NProtoBuf::CustomMerge(src, dst); - // Cerr << "dst2:" << dst.ShortDebugString() << Endl; + // Cerr << "dst2:" << dst.ShortDebugString() << Endl; // repeated uint32 MergeInt = 1; UNIT_ASSERT_EQUAL(dst.MergeIntSize(), 4); diff --git a/library/cpp/protobuf/util/path.cpp b/library/cpp/protobuf/util/path.cpp index efa2a42c8a..aeb9b52b71 100644 --- a/library/cpp/protobuf/util/path.cpp +++ b/library/cpp/protobuf/util/path.cpp @@ -3,59 +3,59 @@ #include <util/generic/yexception.h> namespace NProtoBuf { - TFieldPath::TFieldPath() { - } - - TFieldPath::TFieldPath(const Descriptor* msgType, const TStringBuf& path) { - Init(msgType, path); - } - - TFieldPath::TFieldPath(const TVector<const FieldDescriptor*>& path) - : Path(path) - { - } - - bool TFieldPath::InitUnsafe(const Descriptor* msgType, TStringBuf path) { - Path.clear(); - while (path) { - TStringBuf next; - while (!next && path) - next = path.NextTok('/'); - if (!next) - return true; - - if (!msgType) // need field but no message type - return false; - - TString nextStr(next); - const FieldDescriptor* field = msgType->FindFieldByName(nextStr); - if (!field) { - // Try to find extension field by FindAllExtensions() - const DescriptorPool* pool = msgType->file()->pool(); - Y_ASSERT(pool); // never NULL by protobuf docs - TVector<const FieldDescriptor*> extensions; - pool->FindAllExtensions(msgType, &extensions); // find all extensions of this extendee - for (const FieldDescriptor* ext : extensions) { - if (ext->full_name() == nextStr || ext->name() == nextStr) { - if (field) - return false; // ambiguity - field = ext; - } + TFieldPath::TFieldPath() { + } + + TFieldPath::TFieldPath(const Descriptor* msgType, const TStringBuf& path) { + Init(msgType, path); + } + + TFieldPath::TFieldPath(const TVector<const FieldDescriptor*>& path) + : Path(path) + { + } + + bool TFieldPath::InitUnsafe(const Descriptor* msgType, TStringBuf path) { + Path.clear(); + while (path) { + TStringBuf next; + while (!next && path) + next = path.NextTok('/'); + if (!next) + return true; + + if (!msgType) // need field but no message type + return false; + + TString nextStr(next); + const FieldDescriptor* field = msgType->FindFieldByName(nextStr); + if (!field) { + // Try to find extension field by FindAllExtensions() + const DescriptorPool* pool = msgType->file()->pool(); + Y_ASSERT(pool); // never NULL by protobuf docs + TVector<const FieldDescriptor*> extensions; + pool->FindAllExtensions(msgType, &extensions); // find all extensions of this extendee + for (const FieldDescriptor* ext : extensions) { + if (ext->full_name() == nextStr || ext->name() == nextStr) { + if (field) + return false; // ambiguity + field = ext; + } } } - if (!field) - return false; - - Path.push_back(field); - msgType = field->type() == FieldDescriptor::TYPE_MESSAGE ? field->message_type() : nullptr; - } - return true; - } + if (!field) + return false; - void TFieldPath::Init(const Descriptor* msgType, const TStringBuf& path) { - if (!InitUnsafe(msgType, path)) - ythrow yexception() << "Failed to resolve path \"" << path << "\" relative to " << msgType->full_name(); + Path.push_back(field); + msgType = field->type() == FieldDescriptor::TYPE_MESSAGE ? field->message_type() : nullptr; + } + return true; } + void TFieldPath::Init(const Descriptor* msgType, const TStringBuf& path) { + if (!InitUnsafe(msgType, path)) + ythrow yexception() << "Failed to resolve path \"" << path << "\" relative to " << msgType->full_name(); + } + } diff --git a/library/cpp/protobuf/util/path.h b/library/cpp/protobuf/util/path.h index 487f643a2d..99cf198415 100644 --- a/library/cpp/protobuf/util/path.h +++ b/library/cpp/protobuf/util/path.h @@ -6,47 +6,47 @@ #include <util/generic/vector.h> namespace NProtoBuf { - class TFieldPath { - public: - TFieldPath(); - TFieldPath(const Descriptor* msgType, const TStringBuf& path); // throws exception if path doesn't exist - TFieldPath(const TVector<const FieldDescriptor*>& path); + class TFieldPath { + public: + TFieldPath(); + TFieldPath(const Descriptor* msgType, const TStringBuf& path); // throws exception if path doesn't exist + TFieldPath(const TVector<const FieldDescriptor*>& path); TFieldPath(const TFieldPath&) = default; TFieldPath& operator=(const TFieldPath&) = default; - bool InitUnsafe(const Descriptor* msgType, const TStringBuf path); // noexcept - void Init(const Descriptor* msgType, const TStringBuf& path); // throws + bool InitUnsafe(const Descriptor* msgType, const TStringBuf path); // noexcept + void Init(const Descriptor* msgType, const TStringBuf& path); // throws - const TVector<const FieldDescriptor*>& Fields() const { - return Path; - } + const TVector<const FieldDescriptor*>& Fields() const { + return Path; + } - void AddField(const FieldDescriptor* field) { - Path.push_back(field); - } + void AddField(const FieldDescriptor* field) { + Path.push_back(field); + } - const Descriptor* ParentType() const { - return Empty() ? nullptr : Path.front()->containing_type(); - } + const Descriptor* ParentType() const { + return Empty() ? nullptr : Path.front()->containing_type(); + } - const FieldDescriptor* FieldDescr() const { - return Empty() ? nullptr : Path.back(); - } + const FieldDescriptor* FieldDescr() const { + return Empty() ? nullptr : Path.back(); + } - bool Empty() const { - return Path.empty(); - } + bool Empty() const { + return Path.empty(); + } - explicit operator bool() const { - return !Empty(); - } + explicit operator bool() const { + return !Empty(); + } - bool operator!() const { - return Empty(); - } + bool operator!() const { + return Empty(); + } - private: - TVector<const FieldDescriptor*> Path; - }; + private: + TVector<const FieldDescriptor*> Path; + }; -} +} diff --git a/library/cpp/protobuf/util/pb_io.cpp b/library/cpp/protobuf/util/pb_io.cpp index 6270ee0624..f26b5322c9 100644 --- a/library/cpp/protobuf/util/pb_io.cpp +++ b/library/cpp/protobuf/util/pb_io.cpp @@ -23,50 +23,50 @@ namespace NProtoBuf { void ParseFromBase64String(const TStringBuf dataBase64, Message& m, bool allowUneven) { if (!m.ParseFromString(allowUneven ? Base64DecodeUneven(dataBase64) : Base64StrictDecode(dataBase64))) { - ythrow yexception() << "can't parse " << m.GetTypeName() << " from base64-encoded string"; - } - } + ythrow yexception() << "can't parse " << m.GetTypeName() << " from base64-encoded string"; + } + } bool TryParseFromBase64String(const TStringBuf dataBase64, Message& m, bool allowUneven) { - try { + try { ParseFromBase64String(dataBase64, m, allowUneven); - return true; - } catch (const std::exception&) { - return false; - } + return true; + } catch (const std::exception&) { + return false; + } } - void SerializeToBase64String(const Message& m, TString& dataBase64) { - TString rawData; - if (!m.SerializeToString(&rawData)) { - ythrow yexception() << "can't serialize " << m.GetTypeName(); - } - - Base64EncodeUrl(rawData, dataBase64); + void SerializeToBase64String(const Message& m, TString& dataBase64) { + TString rawData; + if (!m.SerializeToString(&rawData)) { + ythrow yexception() << "can't serialize " << m.GetTypeName(); + } + + Base64EncodeUrl(rawData, dataBase64); } - - TString SerializeToBase64String(const Message& m) { - TString s; - SerializeToBase64String(m, s); - return s; + + TString SerializeToBase64String(const Message& m) { + TString s; + SerializeToBase64String(m, s); + return s; } - bool TrySerializeToBase64String(const Message& m, TString& dataBase64) { - try { - SerializeToBase64String(m, dataBase64); - return true; - } catch (const std::exception&) { - return false; - } + bool TrySerializeToBase64String(const Message& m, TString& dataBase64) { + try { + SerializeToBase64String(m, dataBase64); + return true; + } catch (const std::exception&) { + return false; + } } - const TString ShortUtf8DebugString(const Message& message) { - TextFormat::Printer printer; - printer.SetSingleLineMode(true); - printer.SetUseUtf8StringEscaping(true); - TString result; - printer.PrintToString(message, &result); - return result; + const TString ShortUtf8DebugString(const Message& message) { + TextFormat::Printer printer; + printer.SetSingleLineMode(true); + printer.SetUseUtf8StringEscaping(true); + TString result; + printer.PrintToString(message, &result); + return result; } bool MergePartialFromString(NProtoBuf::Message& m, const TStringBuf serializedProtoMessage) { @@ -81,7 +81,7 @@ namespace NProtoBuf { } } -int operator&(NProtoBuf::Message& m, IBinSaver& f) { +int operator&(NProtoBuf::Message& m, IBinSaver& f) { TStringStream ss; if (f.IsReading()) { f.Add(0, &ss.Str()); diff --git a/library/cpp/protobuf/util/pb_io.h b/library/cpp/protobuf/util/pb_io.h index 493c84cb5f..261adf2b0e 100644 --- a/library/cpp/protobuf/util/pb_io.h +++ b/library/cpp/protobuf/util/pb_io.h @@ -40,9 +40,9 @@ namespace NProtoBuf { bool MergePartialFromString(NProtoBuf::Message& m, const TStringBuf serializedProtoMessage); bool MergeFromString(NProtoBuf::Message& m, const TStringBuf serializedProtoMessage); -} +} -int operator&(NProtoBuf::Message& m, IBinSaver& f); +int operator&(NProtoBuf::Message& m, IBinSaver& f); // Write a textual representation of the given message to the given file. void SerializeToTextFormat(const NProtoBuf::Message& m, const TString& fileName); diff --git a/library/cpp/protobuf/util/pb_io_ut.cpp b/library/cpp/protobuf/util/pb_io_ut.cpp index 875d6dc602..b81274b5ec 100644 --- a/library/cpp/protobuf/util/pb_io_ut.cpp +++ b/library/cpp/protobuf/util/pb_io_ut.cpp @@ -24,7 +24,7 @@ static NProtobufUtilUt::TTextEnumTest GetCorrectEnumMessage() { } static const TString CORRECT_MESSAGE = - R"(Foo: 42 + R"(Foo: 42 )"; static const TString CORRECT_ENUM_NAME_MESSAGE = R"(Slot: EET_SLOT_1 @@ -34,7 +34,7 @@ static const TString CORRECT_ENUM_ID_MESSAGE = )"; static const TString INCORRECT_MESSAGE = - R"(Bar: 1 + R"(Bar: 1 )"; static const TString INCORRECT_ENUM_NAME_MESSAGE = R"(Slot: EET_SLOT_3 @@ -177,8 +177,8 @@ Y_UNIT_TEST_SUITE(TTestProtoBufIO) { NProtobufUtilUt::TTextTest m; const auto f = [&correctFileName](NProtobufUtilUt::TTextTest& mm) { mm = ParseFromTextFormat<NProtobufUtilUt::TTextTest>( - correctFileName, - EParseFromTextFormatOption::AllowUnknownField); + correctFileName, + EParseFromTextFormatOption::AllowUnknownField); }; UNIT_ASSERT_NO_EXCEPTION(f(m)); UNIT_ASSERT(NProtoBuf::IsEqual(GetCorrectMessage(), m)); @@ -188,8 +188,8 @@ Y_UNIT_TEST_SUITE(TTestProtoBufIO) { NProtobufUtilUt::TTextTest m; const auto f = [&incorrectFileName](NProtobufUtilUt::TTextTest& mm) { mm = ParseFromTextFormat<NProtobufUtilUt::TTextTest>( - incorrectFileName, - EParseFromTextFormatOption::AllowUnknownField); + incorrectFileName, + EParseFromTextFormatOption::AllowUnknownField); }; UNIT_ASSERT_NO_EXCEPTION(f(m)); UNIT_ASSERT(NProtoBuf::IsEqual(empty, m)); @@ -354,8 +354,8 @@ Y_UNIT_TEST_SUITE(TTestProtoBufIO) { NProtobufUtilUt::TTextTest m; const auto f = [&correctFileName](NProtobufUtilUt::TTextTest& mm) { mm = MergeFromTextFormat<NProtobufUtilUt::TTextTest>( - correctFileName, - EParseFromTextFormatOption::AllowUnknownField); + correctFileName, + EParseFromTextFormatOption::AllowUnknownField); }; UNIT_ASSERT_NO_EXCEPTION(f(m)); UNIT_ASSERT(NProtoBuf::IsEqual(GetCorrectMessage(), m)); @@ -365,8 +365,8 @@ Y_UNIT_TEST_SUITE(TTestProtoBufIO) { NProtobufUtilUt::TTextTest m; const auto f = [&incorrectFileName](NProtobufUtilUt::TTextTest& mm) { mm = MergeFromTextFormat<NProtobufUtilUt::TTextTest>( - incorrectFileName, - EParseFromTextFormatOption::AllowUnknownField); + incorrectFileName, + EParseFromTextFormatOption::AllowUnknownField); }; UNIT_ASSERT_NO_EXCEPTION(f(m)); UNIT_ASSERT(NProtoBuf::IsEqual(empty, m)); diff --git a/library/cpp/protobuf/util/pb_utils.h b/library/cpp/protobuf/util/pb_utils.h index 9e9a110b48..71a76ff4c2 100644 --- a/library/cpp/protobuf/util/pb_utils.h +++ b/library/cpp/protobuf/util/pb_utils.h @@ -1,11 +1,11 @@ #pragma once #define UPDATE_PB_FIELD_MAX(PBMESS, FIELD, VAL) \ - if ((VAL) > (PBMESS).Get##FIELD()) { \ - (PBMESS).Set##FIELD(VAL); \ - } + if ((VAL) > (PBMESS).Get##FIELD()) { \ + (PBMESS).Set##FIELD(VAL); \ + } -#define UPDATE_OPT_PB_FIELD_MAX(PBMESS, FIELD, VAL) \ - if (!(PBMESS).Has##FIELD() || ((VAL) > (PBMESS).Get##FIELD())) { \ - (PBMESS).Set##FIELD(VAL); \ - } +#define UPDATE_OPT_PB_FIELD_MAX(PBMESS, FIELD, VAL) \ + if (!(PBMESS).Has##FIELD() || ((VAL) > (PBMESS).Get##FIELD())) { \ + (PBMESS).Set##FIELD(VAL); \ + } diff --git a/library/cpp/protobuf/util/repeated_field_utils.h b/library/cpp/protobuf/util/repeated_field_utils.h index c07bd84647..74767eaafe 100644 --- a/library/cpp/protobuf/util/repeated_field_utils.h +++ b/library/cpp/protobuf/util/repeated_field_utils.h @@ -6,8 +6,8 @@ template <typename T> void RemoveRepeatedPtrFieldElement(google::protobuf::RepeatedPtrField<T>* repeated, unsigned index) { google::protobuf::RepeatedPtrField<T> r; - Y_ASSERT(index < (unsigned)repeated->size()); - for (unsigned i = 0; i < (unsigned)repeated->size(); ++i) { + Y_ASSERT(index < (unsigned)repeated->size()); + for (unsigned i = 0; i < (unsigned)repeated->size(); ++i) { if (i == index) { continue; } @@ -17,80 +17,80 @@ void RemoveRepeatedPtrFieldElement(google::protobuf::RepeatedPtrField<T>* repeat } namespace NProtoBuf { - /// Move item to specified position - template <typename TRepeated> - static void MoveRepeatedFieldItem(TRepeated* field, size_t indexFrom, size_t indexTo) { - if (!field->size() || indexFrom >= static_cast<size_t>(field->size()) || indexFrom == indexTo) - return; - if (indexTo >= static_cast<size_t>(field->size())) - indexTo = field->size() - 1; - if (indexFrom > indexTo) { - for (size_t i = indexFrom; i > indexTo; --i) - field->SwapElements(i, i - 1); - } else { - for (size_t i = indexFrom; i < indexTo; ++i) - field->SwapElements(i, i + 1); - } - } + /// Move item to specified position + template <typename TRepeated> + static void MoveRepeatedFieldItem(TRepeated* field, size_t indexFrom, size_t indexTo) { + if (!field->size() || indexFrom >= static_cast<size_t>(field->size()) || indexFrom == indexTo) + return; + if (indexTo >= static_cast<size_t>(field->size())) + indexTo = field->size() - 1; + if (indexFrom > indexTo) { + for (size_t i = indexFrom; i > indexTo; --i) + field->SwapElements(i, i - 1); + } else { + for (size_t i = indexFrom; i < indexTo; ++i) + field->SwapElements(i, i + 1); + } + } - template <typename T> - static T* InsertRepeatedFieldItem(NProtoBuf::RepeatedPtrField<T>* field, size_t index) { - T* ret = field->Add(); - MoveRepeatedFieldItem(field, field->size() - 1, index); - return ret; + template <typename T> + static T* InsertRepeatedFieldItem(NProtoBuf::RepeatedPtrField<T>* field, size_t index) { + T* ret = field->Add(); + MoveRepeatedFieldItem(field, field->size() - 1, index); + return ret; } - template <typename TRepeated> // suitable both for RepeatedField and RepeatedPtrField - static void RemoveRepeatedFieldItem(TRepeated* field, size_t index) { - if ((int)index >= field->size()) - return; + template <typename TRepeated> // suitable both for RepeatedField and RepeatedPtrField + static void RemoveRepeatedFieldItem(TRepeated* field, size_t index) { + if ((int)index >= field->size()) + return; - for (int i = index + 1; i < field->size(); ++i) - field->SwapElements(i - 1, i); + for (int i = index + 1; i < field->size(); ++i) + field->SwapElements(i - 1, i); - field->RemoveLast(); - } + field->RemoveLast(); + } - template <typename TRepeated, typename TPred> // suitable both for RepeatedField and RepeatedPtrField - static void RemoveRepeatedFieldItemIf(TRepeated* repeated, TPred p) { - auto last = std::remove_if(repeated->begin(), repeated->end(), p); - if (last != repeated->end()) { - size_t countToRemove = repeated->end() - last; - while (countToRemove--) - repeated->RemoveLast(); - } - } + template <typename TRepeated, typename TPred> // suitable both for RepeatedField and RepeatedPtrField + static void RemoveRepeatedFieldItemIf(TRepeated* repeated, TPred p) { + auto last = std::remove_if(repeated->begin(), repeated->end(), p); + if (last != repeated->end()) { + size_t countToRemove = repeated->end() - last; + while (countToRemove--) + repeated->RemoveLast(); + } + } - namespace NImpl { - template <typename TRepeated> - static void ShiftLeft(TRepeated* field, int begIndex, int endIndex, size_t shiftSize) { - Y_ASSERT(begIndex <= field->size()); - Y_ASSERT(endIndex <= field->size()); - size_t shiftIndex = (int)shiftSize < begIndex ? begIndex - shiftSize : 0; - for (int i = begIndex; i < endIndex; ++i, ++shiftIndex) - field->SwapElements(shiftIndex, i); - } + namespace NImpl { + template <typename TRepeated> + static void ShiftLeft(TRepeated* field, int begIndex, int endIndex, size_t shiftSize) { + Y_ASSERT(begIndex <= field->size()); + Y_ASSERT(endIndex <= field->size()); + size_t shiftIndex = (int)shiftSize < begIndex ? begIndex - shiftSize : 0; + for (int i = begIndex; i < endIndex; ++i, ++shiftIndex) + field->SwapElements(shiftIndex, i); + } } - // Remove several items at once, could be more efficient compared to calling RemoveRepeatedFieldItem several times + // Remove several items at once, could be more efficient compared to calling RemoveRepeatedFieldItem several times template <typename TRepeated> - static void RemoveRepeatedFieldItems(TRepeated* field, const TVector<size_t>& sortedIndices) { - if (sortedIndices.empty()) - return; + static void RemoveRepeatedFieldItems(TRepeated* field, const TVector<size_t>& sortedIndices) { + if (sortedIndices.empty()) + return; - size_t shift = 1; - for (size_t i = 1; i < sortedIndices.size(); ++i, ++shift) - NImpl::ShiftLeft(field, sortedIndices[i - 1] + 1, sortedIndices[i], shift); - NImpl::ShiftLeft(field, sortedIndices.back() + 1, field->size(), shift); + size_t shift = 1; + for (size_t i = 1; i < sortedIndices.size(); ++i, ++shift) + NImpl::ShiftLeft(field, sortedIndices[i - 1] + 1, sortedIndices[i], shift); + NImpl::ShiftLeft(field, sortedIndices.back() + 1, field->size(), shift); - for (; shift > 0; --shift) - field->RemoveLast(); - } + for (; shift > 0; --shift) + field->RemoveLast(); + } - template <typename TRepeated> - static void ReverseRepeatedFieldItems(TRepeated* field) { - for (int i1 = 0, i2 = field->size() - 1; i1 < i2; ++i1, --i2) - field->SwapElements(i1, i2); - } + template <typename TRepeated> + static void ReverseRepeatedFieldItems(TRepeated* field) { + for (int i1 = 0, i2 = field->size() - 1; i1 < i2; ++i1, --i2) + field->SwapElements(i1, i2); + } } diff --git a/library/cpp/protobuf/util/simple_reflection.cpp b/library/cpp/protobuf/util/simple_reflection.cpp index d842e9ee44..913a011028 100644 --- a/library/cpp/protobuf/util/simple_reflection.cpp +++ b/library/cpp/protobuf/util/simple_reflection.cpp @@ -1,70 +1,70 @@ #include "simple_reflection.h" namespace NProtoBuf { - const Message* GetMessageHelper(const TConstField& curField, bool) { - return curField.HasValue() && curField.IsMessage() ? curField.Get<Message>() : nullptr; - } + const Message* GetMessageHelper(const TConstField& curField, bool) { + return curField.HasValue() && curField.IsMessage() ? curField.Get<Message>() : nullptr; + } - Message* GetMessageHelper(TMutableField& curField, bool createPath) { - if (curField.IsMessage()) { - if (!curField.HasValue()) { - if (createPath) - return curField.Field()->is_repeated() ? curField.AddMessage() : curField.MutableMessage(); - } else { - return curField.MutableMessage(); - } + Message* GetMessageHelper(TMutableField& curField, bool createPath) { + if (curField.IsMessage()) { + if (!curField.HasValue()) { + if (createPath) + return curField.Field()->is_repeated() ? curField.AddMessage() : curField.MutableMessage(); + } else { + return curField.MutableMessage(); + } } - return nullptr; + return nullptr; } - template <class TField, class TMsg> - TMaybe<TField> ByPathImpl(TMsg& msg, const TVector<const FieldDescriptor*>& fieldsPath, bool createPath) { - if (fieldsPath.empty()) - return TMaybe<TField>(); - TMsg* curParent = &msg; - for (size_t i = 0, size = fieldsPath.size(); i < size; ++i) { - const FieldDescriptor* field = fieldsPath[i]; - if (!curParent) - return TMaybe<TField>(); - TField curField(*curParent, field); - if (size - i == 1) // last element in path - return curField; - curParent = GetMessageHelper(curField, createPath); - } - if (curParent) - return TField(*curParent, fieldsPath.back()); - else + template <class TField, class TMsg> + TMaybe<TField> ByPathImpl(TMsg& msg, const TVector<const FieldDescriptor*>& fieldsPath, bool createPath) { + if (fieldsPath.empty()) return TMaybe<TField>(); + TMsg* curParent = &msg; + for (size_t i = 0, size = fieldsPath.size(); i < size; ++i) { + const FieldDescriptor* field = fieldsPath[i]; + if (!curParent) + return TMaybe<TField>(); + TField curField(*curParent, field); + if (size - i == 1) // last element in path + return curField; + curParent = GetMessageHelper(curField, createPath); + } + if (curParent) + return TField(*curParent, fieldsPath.back()); + else + return TMaybe<TField>(); } - TMaybe<TConstField> TConstField::ByPath(const Message& msg, const TVector<const FieldDescriptor*>& fieldsPath) { - return ByPathImpl<TConstField, const Message>(msg, fieldsPath, false); - } - - TMaybe<TConstField> TConstField::ByPath(const Message& msg, const TStringBuf& path) { - TFieldPath fieldPath; - if (!fieldPath.InitUnsafe(msg.GetDescriptor(), path)) - return TMaybe<TConstField>(); - return ByPathImpl<TConstField, const Message>(msg, fieldPath.Fields(), false); - } + TMaybe<TConstField> TConstField::ByPath(const Message& msg, const TVector<const FieldDescriptor*>& fieldsPath) { + return ByPathImpl<TConstField, const Message>(msg, fieldsPath, false); + } - TMaybe<TConstField> TConstField::ByPath(const Message& msg, const TFieldPath& path) { - return ByPathImpl<TConstField, const Message>(msg, path.Fields(), false); - } + TMaybe<TConstField> TConstField::ByPath(const Message& msg, const TStringBuf& path) { + TFieldPath fieldPath; + if (!fieldPath.InitUnsafe(msg.GetDescriptor(), path)) + return TMaybe<TConstField>(); + return ByPathImpl<TConstField, const Message>(msg, fieldPath.Fields(), false); + } - TMaybe<TMutableField> TMutableField::ByPath(Message& msg, const TVector<const FieldDescriptor*>& fieldsPath, bool createPath) { - return ByPathImpl<TMutableField, Message>(msg, fieldsPath, createPath); - } + TMaybe<TConstField> TConstField::ByPath(const Message& msg, const TFieldPath& path) { + return ByPathImpl<TConstField, const Message>(msg, path.Fields(), false); + } - TMaybe<TMutableField> TMutableField::ByPath(Message& msg, const TStringBuf& path, bool createPath) { - TFieldPath fieldPath; - if (!fieldPath.InitUnsafe(msg.GetDescriptor(), path)) - return TMaybe<TMutableField>(); - return ByPathImpl<TMutableField, Message>(msg, fieldPath.Fields(), createPath); - } + TMaybe<TMutableField> TMutableField::ByPath(Message& msg, const TVector<const FieldDescriptor*>& fieldsPath, bool createPath) { + return ByPathImpl<TMutableField, Message>(msg, fieldsPath, createPath); + } - TMaybe<TMutableField> TMutableField::ByPath(Message& msg, const TFieldPath& path, bool createPath) { - return ByPathImpl<TMutableField, Message>(msg, path.Fields(), createPath); - } + TMaybe<TMutableField> TMutableField::ByPath(Message& msg, const TStringBuf& path, bool createPath) { + TFieldPath fieldPath; + if (!fieldPath.InitUnsafe(msg.GetDescriptor(), path)) + return TMaybe<TMutableField>(); + return ByPathImpl<TMutableField, Message>(msg, fieldPath.Fields(), createPath); + } + TMaybe<TMutableField> TMutableField::ByPath(Message& msg, const TFieldPath& path, bool createPath) { + return ByPathImpl<TMutableField, Message>(msg, path.Fields(), createPath); + } + } diff --git a/library/cpp/protobuf/util/simple_reflection.h b/library/cpp/protobuf/util/simple_reflection.h index 61e877a787..172de02f78 100644 --- a/library/cpp/protobuf/util/simple_reflection.h +++ b/library/cpp/protobuf/util/simple_reflection.h @@ -13,137 +13,137 @@ #include <util/system/defaults.h> namespace NProtoBuf { - class TConstField { - public: - TConstField(const Message& msg, const FieldDescriptor* fd) - : Msg(msg) - , Fd(fd) - { - Y_ASSERT(Fd && Fd->containing_type() == Msg.GetDescriptor()); - } - - static TMaybe<TConstField> ByPath(const Message& msg, const TStringBuf& path); - static TMaybe<TConstField> ByPath(const Message& msg, const TVector<const FieldDescriptor*>& fieldsPath); - static TMaybe<TConstField> ByPath(const Message& msg, const TFieldPath& fieldsPath); - - const Message& Parent() const { - return Msg; - } - - const FieldDescriptor* Field() const { - return Fd; - } - - bool HasValue() const { - return IsRepeated() ? Refl().FieldSize(Msg, Fd) > 0 - : Refl().HasField(Msg, Fd); - } - - // deprecated, use HasValue() instead - bool Has() const { - return HasValue(); - } - - size_t Size() const { - return IsRepeated() ? Refl().FieldSize(Msg, Fd) - : (Refl().HasField(Msg, Fd) ? 1 : 0); - } - - template <typename T> - inline typename TSelectCppType<T>::T Get(size_t index = 0) const; - - template <typename TMsg> - inline const TMsg* GetAs(size_t index = 0) const { - // casting version of Get - return IsMessageInstance<TMsg>() ? CheckedCast<const TMsg*>(Get<const Message*>(index)) : nullptr; - } - - template <typename T> - bool IsInstance() const { - return CppType() == TSelectCppType<T>::Result; - } - - template <typename TMsg> - bool IsMessageInstance() const { - return IsMessage() && Fd->message_type() == TMsg::descriptor(); - } - - template <typename TMsg> - bool IsInstance(std::enable_if_t<std::is_base_of<Message, TMsg>::value && !std::is_same<Message, TMsg>::value, void>* = NULL) const { // template will be selected when specifying Message children types - return IsMessage() && Fd->message_type() == TMsg::descriptor(); - } - - bool IsString() const { - return CppType() == FieldDescriptor::CPPTYPE_STRING; - } - - bool IsMessage() const { - return CppType() == FieldDescriptor::CPPTYPE_MESSAGE; - } - - bool HasSameType(const TConstField& other) const { - if (CppType() != other.CppType()) - return false; - if (IsMessage() && Field()->message_type() != other.Field()->message_type()) - return false; - if (CppType() == FieldDescriptor::CPPTYPE_ENUM && Field()->enum_type() != other.Field()->enum_type()) - return false; - return true; - } - - protected: - bool IsRepeated() const { - return Fd->is_repeated(); - } - - FieldDescriptor::CppType CppType() const { - return Fd->cpp_type(); - } - - const Reflection& Refl() const { - return *Msg.GetReflection(); - } + class TConstField { + public: + TConstField(const Message& msg, const FieldDescriptor* fd) + : Msg(msg) + , Fd(fd) + { + Y_ASSERT(Fd && Fd->containing_type() == Msg.GetDescriptor()); + } + + static TMaybe<TConstField> ByPath(const Message& msg, const TStringBuf& path); + static TMaybe<TConstField> ByPath(const Message& msg, const TVector<const FieldDescriptor*>& fieldsPath); + static TMaybe<TConstField> ByPath(const Message& msg, const TFieldPath& fieldsPath); + + const Message& Parent() const { + return Msg; + } + + const FieldDescriptor* Field() const { + return Fd; + } + + bool HasValue() const { + return IsRepeated() ? Refl().FieldSize(Msg, Fd) > 0 + : Refl().HasField(Msg, Fd); + } + + // deprecated, use HasValue() instead + bool Has() const { + return HasValue(); + } + + size_t Size() const { + return IsRepeated() ? Refl().FieldSize(Msg, Fd) + : (Refl().HasField(Msg, Fd) ? 1 : 0); + } + + template <typename T> + inline typename TSelectCppType<T>::T Get(size_t index = 0) const; + + template <typename TMsg> + inline const TMsg* GetAs(size_t index = 0) const { + // casting version of Get + return IsMessageInstance<TMsg>() ? CheckedCast<const TMsg*>(Get<const Message*>(index)) : nullptr; + } + + template <typename T> + bool IsInstance() const { + return CppType() == TSelectCppType<T>::Result; + } + + template <typename TMsg> + bool IsMessageInstance() const { + return IsMessage() && Fd->message_type() == TMsg::descriptor(); + } + + template <typename TMsg> + bool IsInstance(std::enable_if_t<std::is_base_of<Message, TMsg>::value && !std::is_same<Message, TMsg>::value, void>* = NULL) const { // template will be selected when specifying Message children types + return IsMessage() && Fd->message_type() == TMsg::descriptor(); + } + + bool IsString() const { + return CppType() == FieldDescriptor::CPPTYPE_STRING; + } + + bool IsMessage() const { + return CppType() == FieldDescriptor::CPPTYPE_MESSAGE; + } + + bool HasSameType(const TConstField& other) const { + if (CppType() != other.CppType()) + return false; + if (IsMessage() && Field()->message_type() != other.Field()->message_type()) + return false; + if (CppType() == FieldDescriptor::CPPTYPE_ENUM && Field()->enum_type() != other.Field()->enum_type()) + return false; + return true; + } + + protected: + bool IsRepeated() const { + return Fd->is_repeated(); + } + + FieldDescriptor::CppType CppType() const { + return Fd->cpp_type(); + } + + const Reflection& Refl() const { + return *Msg.GetReflection(); + } [[noreturn]] void RaiseUnknown() const { - ythrow yexception() << "Unknown field cpp-type: " << (size_t)CppType(); - } + ythrow yexception() << "Unknown field cpp-type: " << (size_t)CppType(); + } - bool IsSameField(const TConstField& other) const { - return &Parent() == &other.Parent() && Field() == other.Field(); - } + bool IsSameField(const TConstField& other) const { + return &Parent() == &other.Parent() && Field() == other.Field(); + } - protected: - const Message& Msg; - const FieldDescriptor* Fd; - }; + protected: + const Message& Msg; + const FieldDescriptor* Fd; + }; - class TMutableField: public TConstField { - public: - TMutableField(Message& msg, const FieldDescriptor* fd) - : TConstField(msg, fd) - { - } + class TMutableField: public TConstField { + public: + TMutableField(Message& msg, const FieldDescriptor* fd) + : TConstField(msg, fd) + { + } - static TMaybe<TMutableField> ByPath(Message& msg, const TStringBuf& path, bool createPath = false); - static TMaybe<TMutableField> ByPath(Message& msg, const TVector<const FieldDescriptor*>& fieldsPath, bool createPath = false); - static TMaybe<TMutableField> ByPath(Message& msg, const TFieldPath& fieldsPath, bool createPath = false); + static TMaybe<TMutableField> ByPath(Message& msg, const TStringBuf& path, bool createPath = false); + static TMaybe<TMutableField> ByPath(Message& msg, const TVector<const FieldDescriptor*>& fieldsPath, bool createPath = false); + static TMaybe<TMutableField> ByPath(Message& msg, const TFieldPath& fieldsPath, bool createPath = false); - Message* MutableParent() { - return Mut(); - } + Message* MutableParent() { + return Mut(); + } - template <typename T> - inline void Set(T value, size_t index = 0); + template <typename T> + inline void Set(T value, size_t index = 0); - template <typename T> - inline void Add(T value); + template <typename T> + inline void Add(T value); - inline void MergeFrom(const TConstField& src); + inline void MergeFrom(const TConstField& src); - inline void Clear() { - Refl().ClearField(Mut(), Fd); - } - /* + inline void Clear() { + Refl().ClearField(Mut(), Fd); + } + /* void Swap(TMutableField& f) { Y_ASSERT(Field() == f.Field()); @@ -151,139 +151,139 @@ namespace NProtoBuf { // is ported into arcadia protobuf library from up-stream. } */ - inline void RemoveLast() { - Y_ASSERT(HasValue()); - if (IsRepeated()) - Refl().RemoveLast(Mut(), Fd); - else - Clear(); - } - - inline void SwapElements(size_t index1, size_t index2) { - Y_ASSERT(IsRepeated()); - Y_ASSERT(index1 < Size()); - Y_ASSERT(index2 < Size()); - if (index1 == index2) - return; - Refl().SwapElements(Mut(), Fd, index1, index2); - } - - inline void Remove(size_t index) { - if (index >= Size()) - return; - - // Move to the end - for (size_t i = index, size = Size(); i < size - 1; ++i) - SwapElements(i, i + 1); - RemoveLast(); - } - - Message* MutableMessage(size_t index = 0) { - Y_ASSERT(IsMessage()); - if (IsRepeated()) { - Y_ASSERT(index < Size()); - return Refl().MutableRepeatedMessage(Mut(), Fd, index); - } else { - Y_ASSERT(index == 0); - return Refl().MutableMessage(Mut(), Fd); - } - } - - template <typename TMsg> - inline TMsg* AddMessage() { - return CheckedCast<TMsg*>(AddMessage()); - } - - inline Message* AddMessage() { - Y_ASSERT(IsMessage() && IsRepeated()); - return Refl().AddMessage(Mut(), Fd); - } - - private: - Message* Mut() { - return const_cast<Message*>(&Msg); - } - - template <typename T> - inline void MergeValue(T srcValue); - }; - - // template implementations - - template <typename T> - inline typename TSelectCppType<T>::T TConstField::Get(size_t index) const { - Y_ASSERT(index < Size() || !Fd->is_repeated() && index == 0); // Get for single fields is always allowed because of default values -#define TMP_MACRO_FOR_CPPTYPE(CPPTYPE) \ - case CPPTYPE: \ - return CompatCast<CPPTYPE, TSelectCppType<T>::Result>(TSimpleFieldTraits<CPPTYPE>::Get(Msg, Fd, index)); - switch (CppType()) { - APPLY_TMP_MACRO_FOR_ALL_CPPTYPES() - default: - RaiseUnknown(); - } -#undef TMP_MACRO_FOR_CPPTYPE + inline void RemoveLast() { + Y_ASSERT(HasValue()); + if (IsRepeated()) + Refl().RemoveLast(Mut(), Fd); + else + Clear(); + } + + inline void SwapElements(size_t index1, size_t index2) { + Y_ASSERT(IsRepeated()); + Y_ASSERT(index1 < Size()); + Y_ASSERT(index2 < Size()); + if (index1 == index2) + return; + Refl().SwapElements(Mut(), Fd, index1, index2); + } + + inline void Remove(size_t index) { + if (index >= Size()) + return; + + // Move to the end + for (size_t i = index, size = Size(); i < size - 1; ++i) + SwapElements(i, i + 1); + RemoveLast(); + } + + Message* MutableMessage(size_t index = 0) { + Y_ASSERT(IsMessage()); + if (IsRepeated()) { + Y_ASSERT(index < Size()); + return Refl().MutableRepeatedMessage(Mut(), Fd, index); + } else { + Y_ASSERT(index == 0); + return Refl().MutableMessage(Mut(), Fd); + } + } + + template <typename TMsg> + inline TMsg* AddMessage() { + return CheckedCast<TMsg*>(AddMessage()); + } + + inline Message* AddMessage() { + Y_ASSERT(IsMessage() && IsRepeated()); + return Refl().AddMessage(Mut(), Fd); + } + + private: + Message* Mut() { + return const_cast<Message*>(&Msg); + } + + template <typename T> + inline void MergeValue(T srcValue); + }; + + // template implementations + + template <typename T> + inline typename TSelectCppType<T>::T TConstField::Get(size_t index) const { + Y_ASSERT(index < Size() || !Fd->is_repeated() && index == 0); // Get for single fields is always allowed because of default values +#define TMP_MACRO_FOR_CPPTYPE(CPPTYPE) \ + case CPPTYPE: \ + return CompatCast<CPPTYPE, TSelectCppType<T>::Result>(TSimpleFieldTraits<CPPTYPE>::Get(Msg, Fd, index)); + switch (CppType()) { + APPLY_TMP_MACRO_FOR_ALL_CPPTYPES() + default: + RaiseUnknown(); + } +#undef TMP_MACRO_FOR_CPPTYPE } - template <typename T> - inline void TMutableField::Set(T value, size_t index) { - Y_ASSERT(!IsRepeated() && index == 0 || index < Size()); -#define TMP_MACRO_FOR_CPPTYPE(CPPTYPE) \ - case CPPTYPE: \ - TSimpleFieldTraits<CPPTYPE>::Set(*Mut(), Fd, CompatCast<TSelectCppType<T>::Result, CPPTYPE>(value), index); \ - break; - switch (CppType()) { - APPLY_TMP_MACRO_FOR_ALL_CPPTYPES() - default: - RaiseUnknown(); - } -#undef TMP_MACRO_FOR_CPPTYPE + template <typename T> + inline void TMutableField::Set(T value, size_t index) { + Y_ASSERT(!IsRepeated() && index == 0 || index < Size()); +#define TMP_MACRO_FOR_CPPTYPE(CPPTYPE) \ + case CPPTYPE: \ + TSimpleFieldTraits<CPPTYPE>::Set(*Mut(), Fd, CompatCast<TSelectCppType<T>::Result, CPPTYPE>(value), index); \ + break; + switch (CppType()) { + APPLY_TMP_MACRO_FOR_ALL_CPPTYPES() + default: + RaiseUnknown(); + } +#undef TMP_MACRO_FOR_CPPTYPE } - template <typename T> - inline void TMutableField::Add(T value) { -#define TMP_MACRO_FOR_CPPTYPE(CPPTYPE) \ - case CPPTYPE: \ - TSimpleFieldTraits<CPPTYPE>::Add(*Mut(), Fd, CompatCast<TSelectCppType<T>::Result, CPPTYPE>(value)); \ - break; - switch (CppType()) { - APPLY_TMP_MACRO_FOR_ALL_CPPTYPES() - default: - RaiseUnknown(); - } -#undef TMP_MACRO_FOR_CPPTYPE + template <typename T> + inline void TMutableField::Add(T value) { +#define TMP_MACRO_FOR_CPPTYPE(CPPTYPE) \ + case CPPTYPE: \ + TSimpleFieldTraits<CPPTYPE>::Add(*Mut(), Fd, CompatCast<TSelectCppType<T>::Result, CPPTYPE>(value)); \ + break; + switch (CppType()) { + APPLY_TMP_MACRO_FOR_ALL_CPPTYPES() + default: + RaiseUnknown(); + } +#undef TMP_MACRO_FOR_CPPTYPE } - template <typename T> - inline void TMutableField::MergeValue(T srcValue) { + template <typename T> + inline void TMutableField::MergeValue(T srcValue) { Add(srcValue); } - template <> - inline void TMutableField::MergeValue<const Message*>(const Message* srcValue) { - if (IsRepeated()) { - Add(srcValue); - } else { - MutableMessage()->MergeFrom(*srcValue); + template <> + inline void TMutableField::MergeValue<const Message*>(const Message* srcValue) { + if (IsRepeated()) { + Add(srcValue); + } else { + MutableMessage()->MergeFrom(*srcValue); } } - - inline void TMutableField::MergeFrom(const TConstField& src) { - Y_ASSERT(HasSameType(src)); - if (IsSameField(src)) - return; -#define TMP_MACRO_FOR_CPPTYPE(CPPTYPE) \ - case CPPTYPE: { \ - for (size_t itemIdx = 0; itemIdx < src.Size(); ++itemIdx) { \ - MergeValue(TSimpleFieldTraits<CPPTYPE>::Get(src.Parent(), src.Field(), itemIdx)); \ - } \ - break; \ - } - switch (CppType()) { - APPLY_TMP_MACRO_FOR_ALL_CPPTYPES() - default: - RaiseUnknown(); - } -#undef TMP_MACRO_FOR_CPPTYPE - } - + + inline void TMutableField::MergeFrom(const TConstField& src) { + Y_ASSERT(HasSameType(src)); + if (IsSameField(src)) + return; +#define TMP_MACRO_FOR_CPPTYPE(CPPTYPE) \ + case CPPTYPE: { \ + for (size_t itemIdx = 0; itemIdx < src.Size(); ++itemIdx) { \ + MergeValue(TSimpleFieldTraits<CPPTYPE>::Get(src.Parent(), src.Field(), itemIdx)); \ + } \ + break; \ + } + switch (CppType()) { + APPLY_TMP_MACRO_FOR_ALL_CPPTYPES() + default: + RaiseUnknown(); + } +#undef TMP_MACRO_FOR_CPPTYPE + } + } diff --git a/library/cpp/protobuf/util/simple_reflection_ut.cpp b/library/cpp/protobuf/util/simple_reflection_ut.cpp index 169d4703c9..4b6d6e715d 100644 --- a/library/cpp/protobuf/util/simple_reflection_ut.cpp +++ b/library/cpp/protobuf/util/simple_reflection_ut.cpp @@ -36,7 +36,7 @@ Y_UNIT_TEST_SUITE(ProtobufSimpleReflection) { dstOneMsg.MergeFrom(TConstField(src, descr->FindFieldByName("OneMsg"))); UNIT_ASSERT_VALUES_EQUAL(dst.GetOneMsg().RepIntSize(), src.GetOneMsg().RepIntSize()); dstOneMsg.MergeFrom(TConstField(src, descr->FindFieldByName("OneMsg"))); - UNIT_ASSERT_VALUES_EQUAL(dst.GetOneMsg().RepIntSize(), src.GetOneMsg().RepIntSize() * 2); + UNIT_ASSERT_VALUES_EQUAL(dst.GetOneMsg().RepIntSize(), src.GetOneMsg().RepIntSize() * 2); } { // MergeFrom for repeated fields acts like append @@ -44,7 +44,7 @@ Y_UNIT_TEST_SUITE(ProtobufSimpleReflection) { dstRepMsg.MergeFrom(TConstField(src, descr->FindFieldByName("RepMsg"))); UNIT_ASSERT_VALUES_EQUAL(dst.RepMsgSize(), src.RepMsgSize()); dstRepMsg.MergeFrom(TConstField(src, descr->FindFieldByName("RepMsg"))); - UNIT_ASSERT_VALUES_EQUAL(dst.RepMsgSize(), src.RepMsgSize() * 2); + UNIT_ASSERT_VALUES_EQUAL(dst.RepMsgSize(), src.RepMsgSize() * 2); for (size_t repMsgIndex = 0; repMsgIndex < dst.RepMsgSize(); ++repMsgIndex) { UNIT_ASSERT_VALUES_EQUAL(dst.GetRepMsg(repMsgIndex).RepIntSize(), src.GetRepMsg(0).RepIntSize()); } @@ -107,14 +107,14 @@ Y_UNIT_TEST_SUITE(ProtobufSimpleReflection) { msg.AddRepMsg()->AddRepInt(3); UNIT_ASSERT_VALUES_EQUAL(3, msg.RepMsgSize()); // 1, 2, 3 - fld.Remove(1); // from middle + fld.Remove(1); // from middle UNIT_ASSERT_VALUES_EQUAL(2, msg.RepMsgSize()); UNIT_ASSERT_VALUES_EQUAL(1, msg.GetRepMsg(0).GetRepInt(0)); UNIT_ASSERT_VALUES_EQUAL(3, msg.GetRepMsg(1).GetRepInt(0)); msg.AddRepMsg()->AddRepInt(5); UNIT_ASSERT_VALUES_EQUAL(3, msg.RepMsgSize()); // 1, 3, 5 - fld.Remove(2); // from end + fld.Remove(2); // from end UNIT_ASSERT_VALUES_EQUAL(2, msg.RepMsgSize()); UNIT_ASSERT_VALUES_EQUAL(1, msg.GetRepMsg(0).GetRepInt(0)); UNIT_ASSERT_VALUES_EQUAL(3, msg.GetRepMsg(1).GetRepInt(0)); @@ -127,7 +127,7 @@ Y_UNIT_TEST_SUITE(ProtobufSimpleReflection) { msg.AddRepStr("2"); msg.AddRepStr("3"); UNIT_ASSERT_VALUES_EQUAL(3, msg.RepStrSize()); // "1", "2", "3" - fld.Remove(0); // from begin + fld.Remove(0); // from begin UNIT_ASSERT_VALUES_EQUAL(2, msg.RepStrSize()); UNIT_ASSERT_VALUES_EQUAL("2", msg.GetRepStr(0)); UNIT_ASSERT_VALUES_EQUAL("3", msg.GetRepStr(1)); @@ -279,81 +279,81 @@ Y_UNIT_TEST_SUITE(ProtobufSimpleReflection) { // error { - {TSample msg; - UNIT_ASSERT(!TConstField::ByPath(msg, "SomeField")); + {TSample msg; + UNIT_ASSERT(!TConstField::ByPath(msg, "SomeField")); + } + + { + TSample msg; + UNIT_ASSERT(!TMutableField::ByPath(msg, "SomeField/FieldSome")); + } + + { + TSample msg; + UNIT_ASSERT(!TMutableField::ByPath(msg, "SomeField/FieldSome", true)); + } +} + +// extension +{ + TSample msg; + msg.SetExtension(NExt::TTestExt::ExtField, "ext"); + msg.SetExtension(NExt::ExtField, 2); + msg.AddExtension(NExt::Ext2Field, 33); + TInnerSample* subMsg = msg.MutableExtension(NExt::SubMsgExt); + subMsg->AddRepInt(20); + subMsg->SetExtension(NExt::Ext3Field, 54); + + { + TMaybe<TConstField> field = TConstField::ByPath(msg, "NExt.TTestExt.ExtField"); + UNIT_ASSERT(field); + UNIT_ASSERT(field->HasValue()); + UNIT_ASSERT_VALUES_EQUAL("ext", field->Get<TString>()); } - - { - TSample msg; - UNIT_ASSERT(!TMutableField::ByPath(msg, "SomeField/FieldSome")); - } - - { - TSample msg; - UNIT_ASSERT(!TMutableField::ByPath(msg, "SomeField/FieldSome", true)); - } -} - -// extension -{ - TSample msg; - msg.SetExtension(NExt::TTestExt::ExtField, "ext"); - msg.SetExtension(NExt::ExtField, 2); - msg.AddExtension(NExt::Ext2Field, 33); - TInnerSample* subMsg = msg.MutableExtension(NExt::SubMsgExt); - subMsg->AddRepInt(20); - subMsg->SetExtension(NExt::Ext3Field, 54); - - { - TMaybe<TConstField> field = TConstField::ByPath(msg, "NExt.TTestExt.ExtField"); - UNIT_ASSERT(field); - UNIT_ASSERT(field->HasValue()); - UNIT_ASSERT_VALUES_EQUAL("ext", field->Get<TString>()); - } - { - TMaybe<TConstField> field = TConstField::ByPath(msg, "NExt.ExtField"); - UNIT_ASSERT(field); - UNIT_ASSERT(field->HasValue()); - UNIT_ASSERT_VALUES_EQUAL(2, field->Get<int>()); - } - { - TMaybe<TConstField> field = TConstField::ByPath(msg, "ExtField"); // ambiguity - UNIT_ASSERT(!field); - } - { - TMaybe<TConstField> field = TConstField::ByPath(msg, "NExt.Ext2Field"); - UNIT_ASSERT(field); - UNIT_ASSERT(field->HasValue()); - UNIT_ASSERT_VALUES_EQUAL(33, field->Get<int>()); - } - { - TMaybe<TConstField> field = TConstField::ByPath(msg, "Ext2Field"); - UNIT_ASSERT(field); - UNIT_ASSERT(field->HasValue()); - UNIT_ASSERT_VALUES_EQUAL(33, field->Get<int>()); - } - { - TMaybe<TConstField> field = TConstField::ByPath(msg, "SubMsgExt"); - UNIT_ASSERT(field); - UNIT_ASSERT(field->HasValue()); - const TInnerSample* subMsg2 = field->GetAs<TInnerSample>(); - UNIT_ASSERT(subMsg2); - UNIT_ASSERT_VALUES_EQUAL(1, subMsg2->RepIntSize()); - UNIT_ASSERT_VALUES_EQUAL(20, subMsg2->GetRepInt(0)); - UNIT_ASSERT_VALUES_EQUAL(54, subMsg2->GetExtension(NExt::Ext3Field)); - } - { - TMaybe<TConstField> field = TConstField::ByPath(msg, "SubMsgExt/Ext3Field"); - UNIT_ASSERT(field); - UNIT_ASSERT(field->HasValue()); - UNIT_ASSERT_VALUES_EQUAL(54, field->Get<int>()); - } - { - TMaybe<TConstField> field = TConstField::ByPath(msg, "SubMsgExt/RepInt"); - UNIT_ASSERT(field); - UNIT_ASSERT(field->HasValue()); - UNIT_ASSERT_VALUES_EQUAL(20, field->Get<int>()); - } -} -} + { + TMaybe<TConstField> field = TConstField::ByPath(msg, "NExt.ExtField"); + UNIT_ASSERT(field); + UNIT_ASSERT(field->HasValue()); + UNIT_ASSERT_VALUES_EQUAL(2, field->Get<int>()); + } + { + TMaybe<TConstField> field = TConstField::ByPath(msg, "ExtField"); // ambiguity + UNIT_ASSERT(!field); + } + { + TMaybe<TConstField> field = TConstField::ByPath(msg, "NExt.Ext2Field"); + UNIT_ASSERT(field); + UNIT_ASSERT(field->HasValue()); + UNIT_ASSERT_VALUES_EQUAL(33, field->Get<int>()); + } + { + TMaybe<TConstField> field = TConstField::ByPath(msg, "Ext2Field"); + UNIT_ASSERT(field); + UNIT_ASSERT(field->HasValue()); + UNIT_ASSERT_VALUES_EQUAL(33, field->Get<int>()); + } + { + TMaybe<TConstField> field = TConstField::ByPath(msg, "SubMsgExt"); + UNIT_ASSERT(field); + UNIT_ASSERT(field->HasValue()); + const TInnerSample* subMsg2 = field->GetAs<TInnerSample>(); + UNIT_ASSERT(subMsg2); + UNIT_ASSERT_VALUES_EQUAL(1, subMsg2->RepIntSize()); + UNIT_ASSERT_VALUES_EQUAL(20, subMsg2->GetRepInt(0)); + UNIT_ASSERT_VALUES_EQUAL(54, subMsg2->GetExtension(NExt::Ext3Field)); + } + { + TMaybe<TConstField> field = TConstField::ByPath(msg, "SubMsgExt/Ext3Field"); + UNIT_ASSERT(field); + UNIT_ASSERT(field->HasValue()); + UNIT_ASSERT_VALUES_EQUAL(54, field->Get<int>()); + } + { + TMaybe<TConstField> field = TConstField::ByPath(msg, "SubMsgExt/RepInt"); + UNIT_ASSERT(field); + UNIT_ASSERT(field->HasValue()); + UNIT_ASSERT_VALUES_EQUAL(20, field->Get<int>()); + } } +} +} diff --git a/library/cpp/protobuf/util/sort.h b/library/cpp/protobuf/util/sort.h index 985ba6f689..5c9ed37161 100644 --- a/library/cpp/protobuf/util/sort.h +++ b/library/cpp/protobuf/util/sort.h @@ -6,23 +6,23 @@ #include <util/generic/algorithm.h> namespace NProtoBuf { - // TComparePtr is something like: - // typedef bool (*TComparePtr)(const Message* msg1, const Message* msg2); - // typedef bool (*TComparePtr)(const TProto* msg1, const TProto* msg2); + // TComparePtr is something like: + // typedef bool (*TComparePtr)(const Message* msg1, const Message* msg2); + // typedef bool (*TComparePtr)(const TProto* msg1, const TProto* msg2); - template <typename TProto, typename TComparePtr> - void SortMessages(RepeatedPtrField<TProto>& msgs, TComparePtr cmp) { - TVector<TProto*> ptrs; - ptrs.reserve(msgs.size()); - while (msgs.size()) { - ptrs.push_back(msgs.ReleaseLast()); - } + template <typename TProto, typename TComparePtr> + void SortMessages(RepeatedPtrField<TProto>& msgs, TComparePtr cmp) { + TVector<TProto*> ptrs; + ptrs.reserve(msgs.size()); + while (msgs.size()) { + ptrs.push_back(msgs.ReleaseLast()); + } - ::StableSort(ptrs.begin(), ptrs.end(), cmp); - - for (size_t i = 0; i < ptrs.size(); ++i) { - msgs.AddAllocated(ptrs[i]); - } + ::StableSort(ptrs.begin(), ptrs.end(), cmp); + + for (size_t i = 0; i < ptrs.size(); ++i) { + msgs.AddAllocated(ptrs[i]); + } } } diff --git a/library/cpp/protobuf/util/traits.h b/library/cpp/protobuf/util/traits.h index 50f036d0ea..8f00b6d6e1 100644 --- a/library/cpp/protobuf/util/traits.h +++ b/library/cpp/protobuf/util/traits.h @@ -11,302 +11,302 @@ namespace NProtoBuf { #undef GetMessage #endif - struct TCppTypeTraitsBase { - static inline bool Has(const Message& msg, const FieldDescriptor* field) { // non-repeated - return msg.GetReflection()->HasField(msg, field); - } - static inline size_t Size(const Message& msg, const FieldDescriptor* field) { // repeated - return msg.GetReflection()->FieldSize(msg, field); - } - - static inline void Clear(Message& msg, const FieldDescriptor* field) { - msg.GetReflection()->ClearField(&msg, field); - } - - static inline void RemoveLast(Message& msg, const FieldDescriptor* field) { - msg.GetReflection()->RemoveLast(&msg, field); - } - - static inline void SwapElements(Message& msg, const FieldDescriptor* field, int index1, int index2) { - msg.GetReflection()->SwapElements(&msg, field, index1, index2); - } + struct TCppTypeTraitsBase { + static inline bool Has(const Message& msg, const FieldDescriptor* field) { // non-repeated + return msg.GetReflection()->HasField(msg, field); + } + static inline size_t Size(const Message& msg, const FieldDescriptor* field) { // repeated + return msg.GetReflection()->FieldSize(msg, field); + } + + static inline void Clear(Message& msg, const FieldDescriptor* field) { + msg.GetReflection()->ClearField(&msg, field); + } + + static inline void RemoveLast(Message& msg, const FieldDescriptor* field) { + msg.GetReflection()->RemoveLast(&msg, field); + } + + static inline void SwapElements(Message& msg, const FieldDescriptor* field, int index1, int index2) { + msg.GetReflection()->SwapElements(&msg, field, index1, index2); + } + }; + + // default value accessor + template <FieldDescriptor::CppType cpptype> + struct TCppTypeTraitsDefault; + +#define DECLARE_CPPTYPE_DEFAULT(cpptype, method) \ + template <> \ + struct TCppTypeTraitsDefault<cpptype> { \ + static auto GetDefault(const FieldDescriptor* fd) \ + -> decltype(fd->default_value_##method()) { \ + Y_ASSERT(fd); \ + return fd->default_value_##method(); \ + } \ }; - // default value accessor - template <FieldDescriptor::CppType cpptype> - struct TCppTypeTraitsDefault; - -#define DECLARE_CPPTYPE_DEFAULT(cpptype, method) \ - template <> \ - struct TCppTypeTraitsDefault<cpptype> { \ - static auto GetDefault(const FieldDescriptor* fd) \ - -> decltype(fd->default_value_##method()) { \ - Y_ASSERT(fd); \ - return fd->default_value_##method(); \ - } \ - }; - - DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_INT32, int32); - DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_INT64, int64); - DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_UINT32, uint32); - DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_UINT64, uint64); - DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_FLOAT, float); - DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_DOUBLE, double); - DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_BOOL, bool); - DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_ENUM, enum); - DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_STRING, string); + DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_INT32, int32); + DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_INT64, int64); + DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_UINT32, uint32); + DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_UINT64, uint64); + DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_FLOAT, float); + DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_DOUBLE, double); + DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_BOOL, bool); + DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_ENUM, enum); + DECLARE_CPPTYPE_DEFAULT(FieldDescriptor::CppType::CPPTYPE_STRING, string); #undef DECLARE_CPPTYPE_DEFAULT - // getters/setters of field with specified CppType - template <FieldDescriptor::CppType cpptype> - struct TCppTypeTraits : TCppTypeTraitsBase { - static const FieldDescriptor::CppType CppType = cpptype; - - struct T {}; - static T Get(const Message& msg, const FieldDescriptor* field); - static T GetRepeated(const Message& msg, const FieldDescriptor* field, int index); - static T GetDefault(const FieldDescriptor* field); - - static void Set(Message& msg, const FieldDescriptor* field, T value); - static void AddRepeated(Message& msg, const FieldDescriptor* field, T value); - static void SetRepeated(Message& msg, const FieldDescriptor* field, int index, T value); + // getters/setters of field with specified CppType + template <FieldDescriptor::CppType cpptype> + struct TCppTypeTraits : TCppTypeTraitsBase { + static const FieldDescriptor::CppType CppType = cpptype; + + struct T {}; + static T Get(const Message& msg, const FieldDescriptor* field); + static T GetRepeated(const Message& msg, const FieldDescriptor* field, int index); + static T GetDefault(const FieldDescriptor* field); + + static void Set(Message& msg, const FieldDescriptor* field, T value); + static void AddRepeated(Message& msg, const FieldDescriptor* field, T value); + static void SetRepeated(Message& msg, const FieldDescriptor* field, int index, T value); + }; + + // any type T -> CppType + template <typename T> + struct TSelectCppType { + //static const FieldDescriptor::CppType Result = FieldDescriptor::MAX_CPPTYPE; + }; + +#define DECLARE_CPPTYPE_TRAITS(cpptype, type, method) \ + template <> \ + struct TCppTypeTraits<cpptype>: public TCppTypeTraitsBase { \ + typedef type T; \ + static const FieldDescriptor::CppType CppType = cpptype; \ + \ + static inline T Get(const Message& msg, const FieldDescriptor* field) { \ + return msg.GetReflection()->Get##method(msg, field); \ + } \ + static inline T GetRepeated(const Message& msg, const FieldDescriptor* field, int index) { \ + return msg.GetReflection()->GetRepeated##method(msg, field, index); \ + } \ + static inline T GetDefault(const FieldDescriptor* field) { \ + return TCppTypeTraitsDefault<cpptype>::GetDefault(field); \ + } \ + static inline void Set(Message& msg, const FieldDescriptor* field, T value) { \ + msg.GetReflection()->Set##method(&msg, field, value); \ + } \ + static inline void AddRepeated(Message& msg, const FieldDescriptor* field, T value) { \ + msg.GetReflection()->Add##method(&msg, field, value); \ + } \ + static inline void SetRepeated(Message& msg, const FieldDescriptor* field, int index, T value) { \ + msg.GetReflection()->SetRepeated##method(&msg, field, index, value); \ + } \ + }; \ + template <> \ + struct TSelectCppType<type> { \ + static const FieldDescriptor::CppType Result = cpptype; \ + typedef type T; \ }; - // any type T -> CppType - template <typename T> - struct TSelectCppType { - //static const FieldDescriptor::CppType Result = FieldDescriptor::MAX_CPPTYPE; - }; - -#define DECLARE_CPPTYPE_TRAITS(cpptype, type, method) \ - template <> \ - struct TCppTypeTraits<cpptype>: public TCppTypeTraitsBase { \ - typedef type T; \ - static const FieldDescriptor::CppType CppType = cpptype; \ - \ - static inline T Get(const Message& msg, const FieldDescriptor* field) { \ - return msg.GetReflection()->Get##method(msg, field); \ - } \ - static inline T GetRepeated(const Message& msg, const FieldDescriptor* field, int index) { \ - return msg.GetReflection()->GetRepeated##method(msg, field, index); \ - } \ - static inline T GetDefault(const FieldDescriptor* field) { \ - return TCppTypeTraitsDefault<cpptype>::GetDefault(field); \ - } \ - static inline void Set(Message& msg, const FieldDescriptor* field, T value) { \ - msg.GetReflection()->Set##method(&msg, field, value); \ - } \ - static inline void AddRepeated(Message& msg, const FieldDescriptor* field, T value) { \ - msg.GetReflection()->Add##method(&msg, field, value); \ - } \ - static inline void SetRepeated(Message& msg, const FieldDescriptor* field, int index, T value) { \ - msg.GetReflection()->SetRepeated##method(&msg, field, index, value); \ - } \ - }; \ - template <> \ - struct TSelectCppType<type> { \ - static const FieldDescriptor::CppType Result = cpptype; \ - typedef type T; \ - }; - - DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_INT32, i32, Int32); - DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_INT64, i64, Int64); - DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_UINT32, ui32, UInt32); - DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_UINT64, ui64, UInt64); - DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_DOUBLE, double, Double); - DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_FLOAT, float, Float); - DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_BOOL, bool, Bool); - DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_ENUM, const EnumValueDescriptor*, Enum); - DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_STRING, TString, String); - //DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_MESSAGE, const Message&, Message); + DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_INT32, i32, Int32); + DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_INT64, i64, Int64); + DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_UINT32, ui32, UInt32); + DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_UINT64, ui64, UInt64); + DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_DOUBLE, double, Double); + DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_FLOAT, float, Float); + DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_BOOL, bool, Bool); + DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_ENUM, const EnumValueDescriptor*, Enum); + DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_STRING, TString, String); + //DECLARE_CPPTYPE_TRAITS(FieldDescriptor::CPPTYPE_MESSAGE, const Message&, Message); #undef DECLARE_CPPTYPE_TRAITS - // specialization for message pointer - template <> - struct TCppTypeTraits<FieldDescriptor::CPPTYPE_MESSAGE>: public TCppTypeTraitsBase { - typedef const Message* T; - static const FieldDescriptor::CppType CppType = FieldDescriptor::CPPTYPE_MESSAGE; - - static inline T Get(const Message& msg, const FieldDescriptor* field) { - return &(msg.GetReflection()->GetMessage(msg, field)); - } - static inline T GetRepeated(const Message& msg, const FieldDescriptor* field, int index) { - return &(msg.GetReflection()->GetRepeatedMessage(msg, field, index)); - } - static inline Message* Set(Message& msg, const FieldDescriptor* field, const Message* value) { - Message* ret = msg.GetReflection()->MutableMessage(&msg, field); - ret->CopyFrom(*value); - return ret; - } - static inline Message* AddRepeated(Message& msg, const FieldDescriptor* field, const Message* value) { - Message* ret = msg.GetReflection()->AddMessage(&msg, field); - ret->CopyFrom(*value); - return ret; - } - static inline Message* SetRepeated(Message& msg, const FieldDescriptor* field, int index, const Message* value) { - Message* ret = msg.GetReflection()->MutableRepeatedMessage(&msg, field, index); - ret->CopyFrom(*value); - return ret; - } - }; - - template <> - struct TSelectCppType<const Message*> { - static const FieldDescriptor::CppType Result = FieldDescriptor::CPPTYPE_MESSAGE; - typedef const Message* T; - }; - - template <> - struct TSelectCppType<Message> { - static const FieldDescriptor::CppType Result = FieldDescriptor::CPPTYPE_MESSAGE; - typedef const Message* T; - }; - - template <FieldDescriptor::CppType CppType, bool Repeated> - struct TFieldTraits { - typedef TCppTypeTraits<CppType> TBaseTraits; - typedef typename TBaseTraits::T T; - - static inline T Get(const Message& msg, const FieldDescriptor* field, size_t index = 0) { - Y_ASSERT(index == 0); - return TBaseTraits::Get(msg, field); - } - - static inline T GetDefault(const FieldDescriptor* field) { - return TBaseTraits::GetDefault(field); - } - - static inline bool Has(const Message& msg, const FieldDescriptor* field) { - return TBaseTraits::Has(msg, field); - } - - static inline size_t Size(const Message& msg, const FieldDescriptor* field) { - return Has(msg, field); - } - - static inline void Set(Message& msg, const FieldDescriptor* field, T value, size_t index = 0) { - Y_ASSERT(index == 0); - TBaseTraits::Set(msg, field, value); - } - - static inline void Add(Message& msg, const FieldDescriptor* field, T value) { - TBaseTraits::Set(msg, field, value); - } - }; - - template <FieldDescriptor::CppType CppType> - struct TFieldTraits<CppType, true> { - typedef TCppTypeTraits<CppType> TBaseTraits; - typedef typename TBaseTraits::T T; - - static inline T Get(const Message& msg, const FieldDescriptor* field, size_t index = 0) { - return TBaseTraits::GetRepeated(msg, field, index); - } - - static inline T GetDefault(const FieldDescriptor* field) { - return TBaseTraits::GetDefault(field); - } - - static inline size_t Size(const Message& msg, const FieldDescriptor* field) { - return TBaseTraits::Size(msg, field); - } - - static inline bool Has(const Message& msg, const FieldDescriptor* field) { - return Size(msg, field) > 0; - } - - static inline void Set(Message& msg, const FieldDescriptor* field, T value, size_t index = 0) { - TBaseTraits::SetRepeated(msg, field, index, value); - } - - static inline void Add(Message& msg, const FieldDescriptor* field, T value) { - TBaseTraits::AddRepeated(msg, field, value); - } - }; - - // Simpler interface at the cost of checking is_repeated() on each call - template <FieldDescriptor::CppType CppType> - struct TSimpleFieldTraits { - typedef TFieldTraits<CppType, true> TRepeated; - typedef TFieldTraits<CppType, false> TSingle; - typedef typename TRepeated::T T; - - static inline size_t Size(const Message& msg, const FieldDescriptor* field) { - if (field->is_repeated()) - return TRepeated::Size(msg, field); - else - return TSingle::Size(msg, field); - } - - static inline bool Has(const Message& msg, const FieldDescriptor* field) { - if (field->is_repeated()) - return TRepeated::Has(msg, field); - else - return TSingle::Has(msg, field); - } - - static inline T Get(const Message& msg, const FieldDescriptor* field, size_t index = 0) { - Y_ASSERT(index < Size(msg, field) || !field->is_repeated() && index == 0); // Get for single fields is always allowed because of default values - if (field->is_repeated()) - return TRepeated::Get(msg, field, index); - else - return TSingle::Get(msg, field, index); - } - - static inline T GetDefault(const FieldDescriptor* field) { - return TSingle::GetDefault(field); - } - - static inline void Set(Message& msg, const FieldDescriptor* field, T value, size_t index = 0) { - Y_ASSERT(!field->is_repeated() && index == 0 || index < Size(msg, field)); - if (field->is_repeated()) - TRepeated::Set(msg, field, value, index); - else - TSingle::Set(msg, field, value, index); - } - - static inline void Add(Message& msg, const FieldDescriptor* field, T value) { - if (field->is_repeated()) - TRepeated::Add(msg, field, value); - else - TSingle::Add(msg, field, value); - } - }; - - // some cpp-type groups - - template <FieldDescriptor::CppType CppType> - struct TIsIntegerCppType { - enum { - Result = CppType == FieldDescriptor::CPPTYPE_INT32 || - CppType == FieldDescriptor::CPPTYPE_INT64 || - CppType == FieldDescriptor::CPPTYPE_UINT32 || - CppType == FieldDescriptor::CPPTYPE_UINT64 - }; + // specialization for message pointer + template <> + struct TCppTypeTraits<FieldDescriptor::CPPTYPE_MESSAGE>: public TCppTypeTraitsBase { + typedef const Message* T; + static const FieldDescriptor::CppType CppType = FieldDescriptor::CPPTYPE_MESSAGE; + + static inline T Get(const Message& msg, const FieldDescriptor* field) { + return &(msg.GetReflection()->GetMessage(msg, field)); + } + static inline T GetRepeated(const Message& msg, const FieldDescriptor* field, int index) { + return &(msg.GetReflection()->GetRepeatedMessage(msg, field, index)); + } + static inline Message* Set(Message& msg, const FieldDescriptor* field, const Message* value) { + Message* ret = msg.GetReflection()->MutableMessage(&msg, field); + ret->CopyFrom(*value); + return ret; + } + static inline Message* AddRepeated(Message& msg, const FieldDescriptor* field, const Message* value) { + Message* ret = msg.GetReflection()->AddMessage(&msg, field); + ret->CopyFrom(*value); + return ret; + } + static inline Message* SetRepeated(Message& msg, const FieldDescriptor* field, int index, const Message* value) { + Message* ret = msg.GetReflection()->MutableRepeatedMessage(&msg, field, index); + ret->CopyFrom(*value); + return ret; + } + }; + + template <> + struct TSelectCppType<const Message*> { + static const FieldDescriptor::CppType Result = FieldDescriptor::CPPTYPE_MESSAGE; + typedef const Message* T; + }; + + template <> + struct TSelectCppType<Message> { + static const FieldDescriptor::CppType Result = FieldDescriptor::CPPTYPE_MESSAGE; + typedef const Message* T; + }; + + template <FieldDescriptor::CppType CppType, bool Repeated> + struct TFieldTraits { + typedef TCppTypeTraits<CppType> TBaseTraits; + typedef typename TBaseTraits::T T; + + static inline T Get(const Message& msg, const FieldDescriptor* field, size_t index = 0) { + Y_ASSERT(index == 0); + return TBaseTraits::Get(msg, field); + } + + static inline T GetDefault(const FieldDescriptor* field) { + return TBaseTraits::GetDefault(field); + } + + static inline bool Has(const Message& msg, const FieldDescriptor* field) { + return TBaseTraits::Has(msg, field); + } + + static inline size_t Size(const Message& msg, const FieldDescriptor* field) { + return Has(msg, field); + } + + static inline void Set(Message& msg, const FieldDescriptor* field, T value, size_t index = 0) { + Y_ASSERT(index == 0); + TBaseTraits::Set(msg, field, value); + } + + static inline void Add(Message& msg, const FieldDescriptor* field, T value) { + TBaseTraits::Set(msg, field, value); + } + }; + + template <FieldDescriptor::CppType CppType> + struct TFieldTraits<CppType, true> { + typedef TCppTypeTraits<CppType> TBaseTraits; + typedef typename TBaseTraits::T T; + + static inline T Get(const Message& msg, const FieldDescriptor* field, size_t index = 0) { + return TBaseTraits::GetRepeated(msg, field, index); + } + + static inline T GetDefault(const FieldDescriptor* field) { + return TBaseTraits::GetDefault(field); + } + + static inline size_t Size(const Message& msg, const FieldDescriptor* field) { + return TBaseTraits::Size(msg, field); + } + + static inline bool Has(const Message& msg, const FieldDescriptor* field) { + return Size(msg, field) > 0; + } + + static inline void Set(Message& msg, const FieldDescriptor* field, T value, size_t index = 0) { + TBaseTraits::SetRepeated(msg, field, index, value); + } + + static inline void Add(Message& msg, const FieldDescriptor* field, T value) { + TBaseTraits::AddRepeated(msg, field, value); + } + }; + + // Simpler interface at the cost of checking is_repeated() on each call + template <FieldDescriptor::CppType CppType> + struct TSimpleFieldTraits { + typedef TFieldTraits<CppType, true> TRepeated; + typedef TFieldTraits<CppType, false> TSingle; + typedef typename TRepeated::T T; + + static inline size_t Size(const Message& msg, const FieldDescriptor* field) { + if (field->is_repeated()) + return TRepeated::Size(msg, field); + else + return TSingle::Size(msg, field); + } + + static inline bool Has(const Message& msg, const FieldDescriptor* field) { + if (field->is_repeated()) + return TRepeated::Has(msg, field); + else + return TSingle::Has(msg, field); + } + + static inline T Get(const Message& msg, const FieldDescriptor* field, size_t index = 0) { + Y_ASSERT(index < Size(msg, field) || !field->is_repeated() && index == 0); // Get for single fields is always allowed because of default values + if (field->is_repeated()) + return TRepeated::Get(msg, field, index); + else + return TSingle::Get(msg, field, index); + } + + static inline T GetDefault(const FieldDescriptor* field) { + return TSingle::GetDefault(field); + } + + static inline void Set(Message& msg, const FieldDescriptor* field, T value, size_t index = 0) { + Y_ASSERT(!field->is_repeated() && index == 0 || index < Size(msg, field)); + if (field->is_repeated()) + TRepeated::Set(msg, field, value, index); + else + TSingle::Set(msg, field, value, index); + } + + static inline void Add(Message& msg, const FieldDescriptor* field, T value) { + if (field->is_repeated()) + TRepeated::Add(msg, field, value); + else + TSingle::Add(msg, field, value); + } + }; + + // some cpp-type groups + + template <FieldDescriptor::CppType CppType> + struct TIsIntegerCppType { + enum { + Result = CppType == FieldDescriptor::CPPTYPE_INT32 || + CppType == FieldDescriptor::CPPTYPE_INT64 || + CppType == FieldDescriptor::CPPTYPE_UINT32 || + CppType == FieldDescriptor::CPPTYPE_UINT64 + }; }; - template <FieldDescriptor::CppType CppType> - struct TIsFloatCppType { - enum { - Result = CppType == FieldDescriptor::CPPTYPE_FLOAT || - CppType == FieldDescriptor::CPPTYPE_DOUBLE - }; + template <FieldDescriptor::CppType CppType> + struct TIsFloatCppType { + enum { + Result = CppType == FieldDescriptor::CPPTYPE_FLOAT || + CppType == FieldDescriptor::CPPTYPE_DOUBLE + }; }; - template <FieldDescriptor::CppType CppType> - struct TIsNumericCppType { - enum { - Result = CppType == FieldDescriptor::CPPTYPE_BOOL || - TIsIntegerCppType<CppType>::Result || - TIsFloatCppType<CppType>::Result - }; + template <FieldDescriptor::CppType CppType> + struct TIsNumericCppType { + enum { + Result = CppType == FieldDescriptor::CPPTYPE_BOOL || + TIsIntegerCppType<CppType>::Result || + TIsFloatCppType<CppType>::Result + }; }; - // a helper macro for splitting flow by cpp-type (e.g. in a switch) + // a helper macro for splitting flow by cpp-type (e.g. in a switch) -#define APPLY_TMP_MACRO_FOR_ALL_CPPTYPES() \ +#define APPLY_TMP_MACRO_FOR_ALL_CPPTYPES() \ TMP_MACRO_FOR_CPPTYPE(NProtoBuf::FieldDescriptor::CPPTYPE_INT32) \ TMP_MACRO_FOR_CPPTYPE(NProtoBuf::FieldDescriptor::CPPTYPE_INT64) \ TMP_MACRO_FOR_CPPTYPE(NProtoBuf::FieldDescriptor::CPPTYPE_UINT32) \ @@ -317,4 +317,4 @@ namespace NProtoBuf { TMP_MACRO_FOR_CPPTYPE(NProtoBuf::FieldDescriptor::CPPTYPE_ENUM) \ TMP_MACRO_FOR_CPPTYPE(NProtoBuf::FieldDescriptor::CPPTYPE_STRING) \ TMP_MACRO_FOR_CPPTYPE(NProtoBuf::FieldDescriptor::CPPTYPE_MESSAGE) -} +} diff --git a/library/cpp/protobuf/util/walk.h b/library/cpp/protobuf/util/walk.h index d15d76562d..d5d40eee69 100644 --- a/library/cpp/protobuf/util/walk.h +++ b/library/cpp/protobuf/util/walk.h @@ -8,9 +8,9 @@ #include <functional> namespace NProtoBuf { - // Apply @onField processor to each field in @msg (even empty) + // Apply @onField processor to each field in @msg (even empty) // Do not walk deeper the field if the field is an empty message - // Returned bool defines if we should walk down deeper to current node children (true), or not (false) + // Returned bool defines if we should walk down deeper to current node children (true), or not (false) void WalkReflection(Message& msg, std::function<bool(Message&, const FieldDescriptor*)> onField); void WalkReflection(const Message& msg, diff --git a/library/cpp/protobuf/util/walk_ut.cpp b/library/cpp/protobuf/util/walk_ut.cpp index 2ea6071b17..2b101e79c5 100644 --- a/library/cpp/protobuf/util/walk_ut.cpp +++ b/library/cpp/protobuf/util/walk_ut.cpp @@ -7,7 +7,7 @@ using namespace NProtoBuf; Y_UNIT_TEST_SUITE(ProtobufWalk) { - static void InitProto(NProtobufUtilUt::TWalkTest & p, int level = 0) { + static void InitProto(NProtobufUtilUt::TWalkTest & p, int level = 0) { p.SetOptInt(1); p.AddRepInt(2); p.AddRepInt(3); @@ -24,16 +24,16 @@ Y_UNIT_TEST_SUITE(ProtobufWalk) { } } - static bool IncreaseInts(Message & msg, const FieldDescriptor* fd) { + static bool IncreaseInts(Message & msg, const FieldDescriptor* fd) { TMutableField f(msg, fd); if (f.IsInstance<ui32>()) { for (size_t i = 0; i < f.Size(); ++i) - f.Set(f.Get<ui64>(i) + 1, i); // ui64 should be ok! + f.Set(f.Get<ui64>(i) + 1, i); // ui64 should be ok! } return true; } - static bool RepeatString1(Message & msg, const FieldDescriptor* fd) { + static bool RepeatString1(Message & msg, const FieldDescriptor* fd) { TMutableField f(msg, fd); if (f.IsString()) { for (size_t i = 0; i < f.Size(); ++i) @@ -43,7 +43,7 @@ Y_UNIT_TEST_SUITE(ProtobufWalk) { return true; } - static bool ClearXXX(Message & msg, const FieldDescriptor* fd) { + static bool ClearXXX(Message & msg, const FieldDescriptor* fd) { const FieldOptions& opt = fd->options(); if (opt.HasExtension(NProtobufUtilUt::XXX) && opt.GetExtension(NProtobufUtilUt::XXX)) TMutableField(msg, fd).Clear(); diff --git a/library/cpp/protobuf/util/ya.make b/library/cpp/protobuf/util/ya.make index b62028af58..6c9410ee77 100644 --- a/library/cpp/protobuf/util/ya.make +++ b/library/cpp/protobuf/util/ya.make @@ -1,4 +1,4 @@ -LIBRARY() +LIBRARY() OWNER(mowgli) diff --git a/library/cpp/regex/hyperscan/hyperscan.cpp b/library/cpp/regex/hyperscan/hyperscan.cpp index ba321f9c29..ba025c72b1 100644 --- a/library/cpp/regex/hyperscan/hyperscan.cpp +++ b/library/cpp/regex/hyperscan/hyperscan.cpp @@ -255,7 +255,7 @@ namespace NHyperscan { hs_error_t status = Singleton<NPrivate::TImpl>()->SerializeDatabase( db.Get(), &databaseBytes, - &databaseLength); + &databaseLength); TSerializedDatabase serialization(databaseBytes); if (status != HS_SUCCESS) { ythrow yexception() << "Failed to serialize hyperscan database"; @@ -268,7 +268,7 @@ namespace NHyperscan { hs_error_t status = Singleton<NPrivate::TImpl>()->DeserializeDatabase( serialization.begin(), serialization.size(), - &rawDb); + &rawDb); TDatabase db(rawDb); if (status != HS_SUCCESS) { if (status == HS_DB_PLATFORM_ERROR) { diff --git a/library/cpp/regex/hyperscan/hyperscan.h b/library/cpp/regex/hyperscan/hyperscan.h index 1c8f404389..608bc87300 100644 --- a/library/cpp/regex/hyperscan/hyperscan.h +++ b/library/cpp/regex/hyperscan/hyperscan.h @@ -144,7 +144,7 @@ namespace NHyperscan { const TDatabase& db, const TScratch& scratch, const TStringBuf& text, - TCallback& callback // applied to index of matched regex + TCallback& callback // applied to index of matched regex ) { NPrivate::Scan<TCallback>(db, scratch, text, callback, *Singleton<NPrivate::TImpl>()); } @@ -152,7 +152,7 @@ namespace NHyperscan { bool Matches( const TDatabase& db, const TScratch& scratch, - const TStringBuf& text); + const TStringBuf& text); TString Serialize(const TDatabase& db); diff --git a/library/cpp/regex/hyperscan/ut/hyperscan_ut.cpp b/library/cpp/regex/hyperscan/ut/hyperscan_ut.cpp index 9caa53f2e7..9410c8d6ba 100644 --- a/library/cpp/regex/hyperscan/ut/hyperscan_ut.cpp +++ b/library/cpp/regex/hyperscan/ut/hyperscan_ut.cpp @@ -23,14 +23,14 @@ Y_UNIT_TEST_SUITE(HyperscanWrappers) { db, scratch, "abc", - callback); + callback); UNIT_ASSERT_EQUAL(foundId, 0); } Y_UNIT_TEST(Matches) { NHyperscan::TDatabase db = NHyperscan::Compile( "a.c", - HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH); + HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH); NHyperscan::TScratch scratch = NHyperscan::MakeScratch(db); UNIT_ASSERT(NHyperscan::Matches(db, scratch, "abc")); UNIT_ASSERT(!NHyperscan::Matches(db, scratch, "foo")); @@ -49,7 +49,7 @@ Y_UNIT_TEST_SUITE(HyperscanWrappers) { { 42, 241, - }); + }); NHyperscan::TScratch scratch = NHyperscan::MakeScratch(db); UNIT_ASSERT(NHyperscan::Matches(db, scratch, "foo")); @@ -65,7 +65,7 @@ Y_UNIT_TEST_SUITE(HyperscanWrappers) { db, scratch, "fooBaR", - callback); + callback); UNIT_ASSERT_EQUAL(foundIds.size(), 2); UNIT_ASSERT(foundIds.contains(42)); UNIT_ASSERT(foundIds.contains(241)); @@ -82,13 +82,13 @@ Y_UNIT_TEST_SUITE(HyperscanWrappers) { }, { 0, - }); + }); } Y_UNIT_TEST(Serialize) { NHyperscan::TDatabase db = NHyperscan::Compile( "foo", - HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH); + HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH); TString serialization = Serialize(db); db.Reset(); TDatabase db2 = Deserialize(serialization); @@ -101,10 +101,10 @@ Y_UNIT_TEST_SUITE(HyperscanWrappers) { Y_UNIT_TEST(GrowScratch) { NHyperscan::TDatabase db1 = NHyperscan::Compile( "foo", - HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH); + HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH); NHyperscan::TDatabase db2 = NHyperscan::Compile( "longer\\w\\w\\wpattern", - HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH | HS_FLAG_UTF8); + HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH | HS_FLAG_UTF8); NHyperscan::TScratch scratch = NHyperscan::MakeScratch(db1); NHyperscan::GrowScratch(scratch, db2); UNIT_ASSERT(NHyperscan::Matches(db1, scratch, "foo")); @@ -114,7 +114,7 @@ Y_UNIT_TEST_SUITE(HyperscanWrappers) { Y_UNIT_TEST(CloneScratch) { NHyperscan::TDatabase db = NHyperscan::Compile( "foo", - HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH); + HS_FLAG_DOTALL | HS_FLAG_SINGLEMATCH); NHyperscan::TScratch scratch1 = NHyperscan::MakeScratch(db); NHyperscan::TScratch scratch2 = NHyperscan::CloneScratch(scratch1); scratch1.Reset(); diff --git a/library/cpp/regex/pcre/regexp.cpp b/library/cpp/regex/pcre/regexp.cpp index 575c09cee4..e7108ae5e9 100644 --- a/library/cpp/regex/pcre/regexp.cpp +++ b/library/cpp/regex/pcre/regexp.cpp @@ -1,21 +1,21 @@ -#include "regexp.h" - +#include "regexp.h" + #include <util/generic/string.h> #include <util/string/ascii.h> #include <util/system/defaults.h> - + #include <cstdlib> #include <util/generic/noncopyable.h> - + class TGlobalImpl : TNonCopyable { private: - const char* Str; - regmatch_t* Pmatch; + const char* Str; + regmatch_t* Pmatch; int Options; int StrLen; int StartOffset, NotEmptyOpts, MatchPos; int MatchBuf[NMATCHES * 3]; - pcre* PregComp; + pcre* PregComp; enum StateCode { TGI_EXIT, @@ -26,25 +26,25 @@ private: private: void CopyResults(int count) { for (int i = 0; i < count; i++) { - Pmatch[MatchPos].rm_so = MatchBuf[2 * i]; - Pmatch[MatchPos].rm_eo = MatchBuf[2 * i + 1]; + Pmatch[MatchPos].rm_so = MatchBuf[2 * i]; + Pmatch[MatchPos].rm_eo = MatchBuf[2 * i + 1]; MatchPos++; if (MatchPos >= NMATCHES) { ythrow yexception() << "TRegExBase::Exec(): Not enough space in internal buffer."; } - } + } } int DoPcreExec(int opts) { int rc = pcre_exec( - PregComp, /* the compiled pattern */ - nullptr, /* no extra data - we didn't study the pattern */ - Str, /* the subject string */ - StrLen, /* the length of the subject */ - StartOffset, /* start at offset 0 in the subject */ - opts, /* default options */ - MatchBuf, /* output vector for substring information */ - NMATCHES); /* number of elements in the output vector */ + PregComp, /* the compiled pattern */ + nullptr, /* no extra data - we didn't study the pattern */ + Str, /* the subject string */ + StrLen, /* the length of the subject */ + StartOffset, /* start at offset 0 in the subject */ + opts, /* default options */ + MatchBuf, /* output vector for substring information */ + NMATCHES); /* number of elements in the output vector */ if (rc == 0) { ythrow yexception() << "TRegExBase::Exec(): Not enough space in internal buffer."; @@ -55,7 +55,7 @@ private: StateCode CheckEmptyCase() { if (MatchBuf[0] == MatchBuf[1]) { // founded an empty string - if (MatchBuf[0] == StrLen) { // at the end + if (MatchBuf[0] == StrLen) { // at the end return TGI_EXIT; } NotEmptyOpts = PCRE_NOTEMPTY | PCRE_ANCHORED; // trying to find non empty string @@ -65,25 +65,25 @@ private: StateCode CheckNoMatch(int rc) { if (rc == PCRE_ERROR_NOMATCH) { - if (NotEmptyOpts == 0) { + if (NotEmptyOpts == 0) { return TGI_EXIT; } - - MatchBuf[1] = StartOffset + 1; // we have failed to find non-empty-string. trying to find again shifting "previous match offset" + + MatchBuf[1] = StartOffset + 1; // we have failed to find non-empty-string. trying to find again shifting "previous match offset" return TGI_CONTINUE; } return TGI_WALKTHROUGH; } public: - TGlobalImpl(const char* st, regmatch_t& pma, int opts, pcre* pc_re) - : Str(st) - , Pmatch(&pma) - , Options(opts) - , StartOffset(0) - , NotEmptyOpts(0) - , MatchPos(0) - , PregComp(pc_re) + TGlobalImpl(const char* st, regmatch_t& pma, int opts, pcre* pc_re) + : Str(st) + , Pmatch(&pma) + , Options(opts) + , StartOffset(0) + , NotEmptyOpts(0) + , MatchPos(0) + , PregComp(pc_re) { memset(Pmatch, -1, sizeof(regmatch_t) * NMATCHES); StrLen = strlen(Str); @@ -114,29 +114,29 @@ public: return 0; case TGI_WALKTHROUGH: default: - break; - } + break; + } if (rc < 0) { return rc; } CopyResults(rc); - } while (true); + } while (true); - return 0; + return 0; } - + private: }; -class TRegExBaseImpl: public TAtomicRefCount<TRegExBaseImpl> { +class TRegExBaseImpl: public TAtomicRefCount<TRegExBaseImpl> { friend class TRegExBase; protected: - int CompileOptions; + int CompileOptions; TString RegExpr; - regex_t Preg; + regex_t Preg; public: TRegExBaseImpl() @@ -159,7 +159,7 @@ public: } } - int Exec(const char* str, regmatch_t pmatch[], int eflags, int nmatches) const { + int Exec(const char* str, regmatch_t pmatch[], int eflags, int nmatches) const { if (!RegExpr) { ythrow yexception() << "Regular expression is not compiled"; } @@ -170,12 +170,12 @@ public: return regexec(&Preg, str, nmatches, pmatch, eflags); } else { int options = 0; - if ((eflags & REG_NOTBOL) != 0) - options |= PCRE_NOTBOL; - if ((eflags & REG_NOTEOL) != 0) - options |= PCRE_NOTEOL; + if ((eflags & REG_NOTBOL) != 0) + options |= PCRE_NOTBOL; + if ((eflags & REG_NOTEOL) != 0) + options |= PCRE_NOTEOL; - return TGlobalImpl(str, pmatch[0], options, (pcre*)Preg.re_pcre).ExecGlobal(); + return TGlobalImpl(str, pmatch[0], options, (pcre*)Preg.re_pcre).ExecGlobal(); } } @@ -195,12 +195,12 @@ bool TRegExBase::IsCompiled() const { return Impl && Impl->IsCompiled(); } -TRegExBase::TRegExBase(const char* re, int cflags) { +TRegExBase::TRegExBase(const char* re, int cflags) { if (re) { Compile(re, cflags); } } - + TRegExBase::TRegExBase(const TString& re, int cflags) { Compile(re, cflags); } @@ -211,8 +211,8 @@ TRegExBase::~TRegExBase() { void TRegExBase::Compile(const TString& re, int cflags) { Impl = new TRegExBaseImpl(re, cflags); } - -int TRegExBase::Exec(const char* str, regmatch_t pmatch[], int eflags, int nmatches) const { + +int TRegExBase::Exec(const char* str, regmatch_t pmatch[], int eflags, int nmatches) const { if (!Impl) ythrow yexception() << "!Regular expression is not compiled"; return Impl->Exec(str, pmatch, eflags, nmatches); @@ -230,22 +230,22 @@ TString TRegExBase::GetRegExpr() const { return Impl->RegExpr; } -TRegExMatch::TRegExMatch(const char* re, int cflags) - : TRegExBase(re, cflags) -{ -} +TRegExMatch::TRegExMatch(const char* re, int cflags) + : TRegExBase(re, cflags) +{ +} TRegExMatch::TRegExMatch(const TString& re, int cflags) : TRegExBase(re, cflags) { } -bool TRegExMatch::Match(const char* str) const { +bool TRegExMatch::Match(const char* str) const { return Exec(str, nullptr, 0, 0) == 0; } -TRegExSubst::TRegExSubst(const char* re, int cflags) - : TRegExBase(re, cflags) +TRegExSubst::TRegExSubst(const char* re, int cflags) + : TRegExBase(re, cflags) , Replacement(nullptr) { memset(Brfs, 0, sizeof(TBackReferences) * NMATCHES); @@ -256,7 +256,7 @@ TString TRegExSubst::Replace(const char* str, int eflags) { if (BrfsCount) { if (Exec(str, PMatch, eflags) == 0) { int i; - for (i = 0; i < BrfsCount; i++) { + for (i = 0; i < BrfsCount; i++) { s += TString(Replacement, Brfs[i].Beg, Brfs[i].End - Brfs[i].Beg); if (Brfs[i].Refer >= 0 && Brfs[i].Refer < NMATCHES) s += TString(str, PMatch[Brfs[i].Refer].rm_so, int(PMatch[Brfs[i].Refer].rm_eo - PMatch[Brfs[i].Refer].rm_so)); @@ -280,15 +280,15 @@ TString TRegExSubst::Replace(const char* str, int eflags) { // {beg = 22, end = 25, Refer = -1} => "ccc" // {beg = 0, end = 0, Refer = 0} //*** -int TRegExSubst::ParseReplacement(const char* repl) { +int TRegExSubst::ParseReplacement(const char* repl) { Replacement = repl; if (!Replacement || *Replacement == 0) return 0; - char* pos = (char*)Replacement; + char* pos = (char*)Replacement; char* pos1 = nullptr; char* pos2 = nullptr; int i = 0; - while (pos && *pos && i < NMATCHES) { + while (pos && *pos && i < NMATCHES) { pos1 = strchr(pos, '$'); Brfs[i].Refer = -1; pos2 = pos1; @@ -296,11 +296,11 @@ int TRegExSubst::ParseReplacement(const char* repl) { pos2 = pos1 + 1; while (IsAsciiDigit(*pos2)) pos2++; - if (pos2 > pos1 + 1) { + if (pos2 > pos1 + 1) { Brfs[i].Refer = atol(TString(Replacement, pos1 + 1 - Replacement, pos2 - (pos1 + 1)).data()); } else { pos1++; - if (*pos2 == '$') + if (*pos2 == '$') pos2++; Brfs[i].Refer = -1; } diff --git a/library/cpp/regex/pcre/regexp.h b/library/cpp/regex/pcre/regexp.h index bc610bd2f3..c74d20b3ad 100644 --- a/library/cpp/regex/pcre/regexp.h +++ b/library/cpp/regex/pcre/regexp.h @@ -1,16 +1,16 @@ #pragma once #include <sys/types.h> - + #include <util/system/defaults.h> #include <util/generic/string.h> #include <util/generic/yexception.h> - -#include <contrib/libs/pcre/pcre.h> -#include <contrib/libs/pcre/pcreposix.h> - + +#include <contrib/libs/pcre/pcre.h> +#include <contrib/libs/pcre/pcreposix.h> + //THIS CODE LOOKS LIKE A TRASH, BUT WORKS. - + #define NMATCHES 100 #define REGEXP_GLOBAL 0x0080 // use this if you want to find all occurences @@ -19,38 +19,38 @@ class TRegExBaseImpl; class TRegExBase { protected: TSimpleIntrusivePtr<TRegExBaseImpl> Impl; - + public: TRegExBase(const char* regExpr = nullptr, int cflags = REG_EXTENDED); TRegExBase(const TString& regExpr, int cflags = REG_EXTENDED); virtual ~TRegExBase(); - int Exec(const char* str, regmatch_t pmatch[], int eflags, int nmatches = NMATCHES) const; + int Exec(const char* str, regmatch_t pmatch[], int eflags, int nmatches = NMATCHES) const; void Compile(const TString& regExpr, int cflags = REG_EXTENDED); bool IsCompiled() const; int GetCompileOptions() const; TString GetRegExpr() const; }; -class TRegExMatch: public TRegExBase { +class TRegExMatch: public TRegExBase { public: TRegExMatch(const char* regExpr = nullptr, int cflags = REG_NOSUB | REG_EXTENDED); TRegExMatch(const TString& regExpr, int cflags = REG_NOSUB | REG_EXTENDED); - bool Match(const char* str) const; + bool Match(const char* str) const; }; -struct TBackReferences { +struct TBackReferences { int Beg; int End; int Refer; }; -class TRegExSubst: public TRegExBase { +class TRegExSubst: public TRegExBase { private: const char* Replacement; - regmatch_t PMatch[NMATCHES]; + regmatch_t PMatch[NMATCHES]; TBackReferences Brfs[NMATCHES]; int BrfsCount; @@ -59,5 +59,5 @@ public: TRegExSubst(const char* regExpr = nullptr, int cflags = REG_EXTENDED); TString Replace(const char* str, int eflags = 0); - int ParseReplacement(const char* replacement); + int ParseReplacement(const char* replacement); }; diff --git a/library/cpp/regex/pcre/regexp_ut.cpp b/library/cpp/regex/pcre/regexp_ut.cpp index 5184e801cc..6ace430a16 100644 --- a/library/cpp/regex/pcre/regexp_ut.cpp +++ b/library/cpp/regex/pcre/regexp_ut.cpp @@ -11,17 +11,17 @@ struct TRegTest { int CompileOptions; int RunOptions; - TRegTest(const char* re, const char* text, const char* res, int copts = REG_EXTENDED, int ropts = 0) - : Regexp(re) - , Data(text) - , Result(res) - , CompileOptions(copts) - , RunOptions(ropts) - { - } + TRegTest(const char* re, const char* text, const char* res, int copts = REG_EXTENDED, int ropts = 0) + : Regexp(re) + , Data(text) + , Result(res) + , CompileOptions(copts) + , RunOptions(ropts) + { + } }; -struct TSubstTest: public TRegTest { +struct TSubstTest: public TRegTest { const char* Replacement; const char* Replacement2; @@ -29,15 +29,15 @@ struct TSubstTest: public TRegTest { : TRegTest(re, text, res, REG_EXTENDED, REGEXP_GLOBAL) , Replacement(repl) , Replacement2(repl2) - { - } + { + } }; -const TRegTest REGTEST_DATA[] = { - TRegTest("test", "its a test and test string.", "6 10", REG_EXTENDED, 0), +const TRegTest REGTEST_DATA[] = { + TRegTest("test", "its a test and test string.", "6 10", REG_EXTENDED, 0), TRegTest("test", "its a test and test string.", "6 10 15 19", REG_EXTENDED, REGEXP_GLOBAL), TRegTest("test|[an]{0,0}", "test and test an test string tes", "0 4 4 4 5 5 6 6 7 7 8 8 9 13 13 13 14 14 15 15 16 16 17 21 21 21 22 22 23 23 24 24 25 25 26 26 27 27 28 28 29 29 30 30 31 31 32 32", REG_EXTENDED, REGEXP_GLOBAL), - TRegTest("test[an]{1,}", "test and test an test string tes", "NM", REG_EXTENDED, REGEXP_GLOBAL)}; + TRegTest("test[an]{1,}", "test and test an test string tes", "NM", REG_EXTENDED, REGEXP_GLOBAL)}; const TSubstTest SUBSTTEST_DATA[] = { TSubstTest("([a-zA-Z]*[0-9]+) (_[a-z]+)", "Xxx123 534 ___124 bsd _A ZXC _L 141 _sd dsfg QWE123 _bbb", "141 XXX/_sd", "$1 XXX/$2", "$2$2$2 YY$1Y/$2")}; @@ -48,7 +48,7 @@ private: private: UNIT_TEST_SUITE(TRegexpTest); - UNIT_TEST(TestRe) + UNIT_TEST(TestRe) UNIT_TEST(TestSubst) UNIT_TEST(TestOffEndOfBuffer); UNIT_TEST_SUITE_END(); diff --git a/library/cpp/regex/pcre/ya.make b/library/cpp/regex/pcre/ya.make index d34911f103..4971c6f35a 100644 --- a/library/cpp/regex/pcre/ya.make +++ b/library/cpp/regex/pcre/ya.make @@ -1,4 +1,4 @@ -LIBRARY() +LIBRARY() OWNER(g:util) diff --git a/library/cpp/regex/pire/extraencodings.cpp b/library/cpp/regex/pire/extraencodings.cpp index 2e507e4b67..8645d6cd4f 100644 --- a/library/cpp/regex/pire/extraencodings.cpp +++ b/library/cpp/regex/pire/extraencodings.cpp @@ -8,73 +8,73 @@ #include "pire.h" namespace NPire { - namespace { - // A one-byte encoding which is capable of transforming upper half of the character - // table to/from Unicode chars. - class TOneByte: public TEncoding { - public: - TOneByte(ECharset doccode) { - Table_ = CodePageByCharset(doccode)->unicode; - for (size_t i = 0; i < 256; ++i) - Reverse_.insert(std::make_pair(Table_[i], static_cast<char>(i))); - } + namespace { + // A one-byte encoding which is capable of transforming upper half of the character + // table to/from Unicode chars. + class TOneByte: public TEncoding { + public: + TOneByte(ECharset doccode) { + Table_ = CodePageByCharset(doccode)->unicode; + for (size_t i = 0; i < 256; ++i) + Reverse_.insert(std::make_pair(Table_[i], static_cast<char>(i))); + } - wchar32 FromLocal(const char*& begin, const char* end) const override { - if (begin != end) - return Table_[static_cast<unsigned char>(*begin++)]; - else - ythrow yexception() << "EOF reached in Pire::OneByte::fromLocal()"; - } + wchar32 FromLocal(const char*& begin, const char* end) const override { + if (begin != end) + return Table_[static_cast<unsigned char>(*begin++)]; + else + ythrow yexception() << "EOF reached in Pire::OneByte::fromLocal()"; + } - TString ToLocal(wchar32 c) const override { - THashMap<wchar32, char>::const_iterator i = Reverse_.find(c); - if (i != Reverse_.end()) - return TString(1, i->second); - else - return TString(); - } + TString ToLocal(wchar32 c) const override { + THashMap<wchar32, char>::const_iterator i = Reverse_.find(c); + if (i != Reverse_.end()) + return TString(1, i->second); + else + return TString(); + } - void AppendDot(TFsm& fsm) const override { - fsm.AppendDot(); - } + void AppendDot(TFsm& fsm) const override { + fsm.AppendDot(); + } - private: - const wchar32* Table_; - THashMap<wchar32, char> Reverse_; - }; + private: + const wchar32* Table_; + THashMap<wchar32, char> Reverse_; + }; - template <unsigned N> - struct TOneByteHelper: public TOneByte { - inline TOneByteHelper() - : TOneByte((ECharset)N) - { - } - }; - } + template <unsigned N> + struct TOneByteHelper: public TOneByte { + inline TOneByteHelper() + : TOneByte((ECharset)N) + { + } + }; + } - namespace NEncodings { - const NPire::TEncoding& Koi8r() { - return *Singleton<TOneByteHelper<CODES_KOI8>>(); - } + namespace NEncodings { + const NPire::TEncoding& Koi8r() { + return *Singleton<TOneByteHelper<CODES_KOI8>>(); + } - const NPire::TEncoding& Cp1251() { - return *Singleton<TOneByteHelper<CODES_WIN>>(); + const NPire::TEncoding& Cp1251() { + return *Singleton<TOneByteHelper<CODES_WIN>>(); } - const NPire::TEncoding& Get(ECharset encoding) { - switch (encoding) { - case CODES_WIN: - return Cp1251(); - case CODES_KOI8: - return Koi8r(); - case CODES_ASCII: - return NPire::NEncodings::Latin1(); - case CODES_UTF8: - return NPire::NEncodings::Utf8(); - default: - ythrow yexception() << "Pire::Encodings::get(ECharset): unknown encoding " << (int)encoding; - } - } + const NPire::TEncoding& Get(ECharset encoding) { + switch (encoding) { + case CODES_WIN: + return Cp1251(); + case CODES_KOI8: + return Koi8r(); + case CODES_ASCII: + return NPire::NEncodings::Latin1(); + case CODES_UTF8: + return NPire::NEncodings::Utf8(); + default: + ythrow yexception() << "Pire::Encodings::get(ECharset): unknown encoding " << (int)encoding; + } + } } diff --git a/library/cpp/regex/pire/inline/ya.make b/library/cpp/regex/pire/inline/ya.make index d4850f7b45..5a83468746 100644 --- a/library/cpp/regex/pire/inline/ya.make +++ b/library/cpp/regex/pire/inline/ya.make @@ -6,8 +6,8 @@ OWNER( g:util davenger ) - -PEERDIR( + +PEERDIR( ADDINCL library/cpp/regex/pire ) diff --git a/library/cpp/regex/pire/pcre2pire.cpp b/library/cpp/regex/pire/pcre2pire.cpp index f788beb85f..498a8abc25 100644 --- a/library/cpp/regex/pire/pcre2pire.cpp +++ b/library/cpp/regex/pire/pcre2pire.cpp @@ -2,7 +2,7 @@ #include <util/generic/vector.h> #include <util/generic/yexception.h> -TString Pcre2Pire(const TString& src) { +TString Pcre2Pire(const TString& src) { TVector<char> result; result.reserve(src.size() + 1); diff --git a/library/cpp/regex/pire/pcre2pire.h b/library/cpp/regex/pire/pcre2pire.h index 46e45b9193..b4d3b34205 100644 --- a/library/cpp/regex/pire/pcre2pire.h +++ b/library/cpp/regex/pire/pcre2pire.h @@ -1,5 +1,5 @@ -#pragma once - +#pragma once + // Author: smikler@yandex-team.ru #include <util/generic/string.h> diff --git a/library/cpp/regex/pire/pire.h b/library/cpp/regex/pire/pire.h index 286fecd693..148301f39d 100644 --- a/library/cpp/regex/pire/pire.h +++ b/library/cpp/regex/pire/pire.h @@ -41,9 +41,9 @@ namespace NPire { using TError = Pire::Error; // Helper functions - using Pire::LongestPrefix; - using Pire::LongestSuffix; - using Pire::Matches; + using Pire::LongestPrefix; + using Pire::LongestSuffix; + using Pire::Matches; using Pire::MmappedScanner; using Pire::Run; using Pire::Runner; @@ -55,8 +55,8 @@ namespace NPire { using namespace Pire::Consts; namespace NFeatures { - using Pire::Features::AndNotSupport; - using Pire::Features::Capture; + using Pire::Features::AndNotSupport; + using Pire::Features::Capture; using Pire::Features::CaseInsensitive; using Pire::Features::GlueSimilarGlyphs; } @@ -65,8 +65,8 @@ namespace NPire { using Pire::Encodings::Latin1; using Pire::Encodings::Utf8; - const NPire::TEncoding& Koi8r(); - const NPire::TEncoding& Cp1251(); + const NPire::TEncoding& Koi8r(); + const NPire::TEncoding& Cp1251(); const NPire::TEncoding& Get(ECharset encoding); } diff --git a/library/cpp/regex/pire/regexp.h b/library/cpp/regex/pire/regexp.h index 94bba4064b..d5424e359a 100644 --- a/library/cpp/regex/pire/regexp.h +++ b/library/cpp/regex/pire/regexp.h @@ -1,7 +1,7 @@ #pragma once - -#include "pire.h" - + +#include "pire.h" + #include <library/cpp/charset/doccodes.h> #include <library/cpp/charset/recyr.hh> #include <util/generic/maybe.h> @@ -10,26 +10,26 @@ #include <util/generic/vector.h> #include <util/generic/yexception.h> -namespace NRegExp { +namespace NRegExp { struct TMatcher; - + struct TFsmBase { struct TOptions { inline TOptions& SetCaseInsensitive(bool v) noexcept { CaseInsensitive = v; return *this; } - + inline TOptions& SetSurround(bool v) noexcept { Surround = v; return *this; } - + inline TOptions& SetCapture(size_t pos) noexcept { CapturePos = pos; return *this; - } - + } + inline TOptions& SetCharset(ECharset charset) noexcept { Charset = charset; return *this; @@ -68,64 +68,64 @@ namespace NRegExp { if (opts.CaseInsensitive) { lexer.AddFeature(NPire::NFeatures::CaseInsensitive()); - } - + } + if (opts.CapturePos) { lexer.AddFeature(NPire::NFeatures::Capture(*opts.CapturePos)); - } - + } + if (opts.AndNotSupport) { lexer.AddFeature(NPire::NFeatures::AndNotSupport()); } switch (opts.Charset) { - case CODES_UNKNOWN: - break; - case CODES_UTF8: - lexer.SetEncoding(NPire::NEncodings::Utf8()); - break; - case CODES_KOI8: - lexer.SetEncoding(NPire::NEncodings::Koi8r()); - break; - default: - lexer.SetEncoding(NPire::NEncodings::Get(opts.Charset)); - break; + case CODES_UNKNOWN: + break; + case CODES_UTF8: + lexer.SetEncoding(NPire::NEncodings::Utf8()); + break; + case CODES_KOI8: + lexer.SetEncoding(NPire::NEncodings::Koi8r()); + break; + default: + lexer.SetEncoding(NPire::NEncodings::Get(opts.Charset)); + break; } NPire::TFsm ret = lexer.Parse(); if (opts.Surround) { ret.Surround(); - } - + } + if (needDetermine) { ret.Determine(); } - + return ret; } }; - + template <class TScannerType> class TFsmParser: public TFsmBase { public: typedef TScannerType TScanner; - + public: inline explicit TFsmParser(const TStringBuf& regexp, const TOptions& opts = TOptions(), bool needDetermine = true) : Scanner(Parse(regexp, opts, needDetermine).template Compile<TScanner>()) { } - + inline const TScanner& GetScanner() const noexcept { return Scanner; } - + static inline TFsmParser False() { return TFsmParser(NPire::TFsm::MakeFalse().Compile<TScanner>()); } - + inline explicit TFsmParser(const TScanner& compiled) : Scanner(compiled) { @@ -135,12 +135,12 @@ namespace NRegExp { private: TScanner Scanner; - }; - + }; + class TFsm: public TFsmParser<NPire::TNonrelocScanner> { public: inline explicit TFsm(const TStringBuf& regexp, - const TOptions& opts = TOptions()) + const TOptions& opts = TOptions()) : TFsmParser<TScanner>(regexp, opts) { } @@ -150,7 +150,7 @@ namespace NRegExp { { } - static inline TFsm Glue(const TFsm& l, const TFsm& r) { + static inline TFsm Glue(const TFsm& l, const TFsm& r) { return TFsm(TScanner::Glue(l.GetScanner(), r.GetScanner())); } @@ -160,23 +160,23 @@ namespace NRegExp { } }; - static inline TFsm operator|(const TFsm& l, const TFsm& r) { - return TFsm::Glue(l, r); - } - - struct TCapturingFsm : TFsmParser<NPire::TCapturingScanner> { + static inline TFsm operator|(const TFsm& l, const TFsm& r) { + return TFsm::Glue(l, r); + } + + struct TCapturingFsm : TFsmParser<NPire::TCapturingScanner> { inline explicit TCapturingFsm(const TStringBuf& regexp, - TOptions opts = TOptions()) + TOptions opts = TOptions()) : TFsmParser<TScanner>(regexp, - opts.SetSurround(true).CapturePos ? opts : opts.SetCapture(1)) { + opts.SetSurround(true).CapturePos ? opts : opts.SetCapture(1)) { } - + inline TCapturingFsm(const TFsmParser<TScanner>& fsm) : TFsmParser<TScanner>(fsm) { } }; - + struct TSlowCapturingFsm : TFsmParser<NPire::TSlowCapturingScanner> { inline explicit TSlowCapturingFsm(const TStringBuf& regexp, TOptions opts = TOptions()) @@ -194,43 +194,43 @@ namespace NRegExp { class TMatcherBase { public: typedef typename TFsm::TScanner::State TState; - + public: inline explicit TMatcherBase(const TFsm& fsm) : Fsm(fsm) { Fsm.GetScanner().Initialize(State); } - + inline bool Final() const noexcept { return GetScanner().Final(GetState()); } - + protected: inline void Run(const char* data, size_t len, bool addBegin, bool addEnd) noexcept { if (addBegin) { NPire::Step(GetScanner(), State, NPire::BeginMark); - } + } NPire::Run(GetScanner(), State, data, data + len); if (addEnd) { NPire::Step(GetScanner(), State, NPire::EndMark); } } - + inline const typename TFsm::TScanner& GetScanner() const noexcept { return Fsm.GetScanner(); } - + inline const TState& GetState() const noexcept { return State; } - + private: const TFsm& Fsm; TState State; - }; + }; - struct TMatcher : TMatcherBase<TFsm> { + struct TMatcher : TMatcherBase<TFsm> { inline explicit TMatcher(const TFsm& fsm) : TMatcherBase<TFsm>(fsm) { @@ -334,4 +334,4 @@ namespace NRegExp { return *this; } }; -} +} diff --git a/library/cpp/regex/pire/ut/regexp_ut.cpp b/library/cpp/regex/pire/ut/regexp_ut.cpp index e7206de9ad..294bc65fa7 100644 --- a/library/cpp/regex/pire/ut/regexp_ut.cpp +++ b/library/cpp/regex/pire/ut/regexp_ut.cpp @@ -1,21 +1,21 @@ #include <library/cpp/testing/unittest/registar.h> - + #include <library/cpp/regex/pire/regexp.h> #include <library/cpp/regex/pire/pcre2pire.h> - + Y_UNIT_TEST_SUITE(TRegExp) { - using namespace NRegExp; - + using namespace NRegExp; + Y_UNIT_TEST(False) { - UNIT_ASSERT(!TMatcher(TFsm::False()).Match("").Final()); + UNIT_ASSERT(!TMatcher(TFsm::False()).Match("").Final()); UNIT_ASSERT(!TMatcher(TFsm::False()).Match(TStringBuf{}).Final()); - } - + } + Y_UNIT_TEST(Surround) { - UNIT_ASSERT(TMatcher(TFsm("qw", TFsm::TOptions().SetSurround(true))).Match("aqwb").Final()); - UNIT_ASSERT(!TMatcher(TFsm("qw", TFsm::TOptions().SetSurround(false))).Match("aqwb").Final()); - } - + UNIT_ASSERT(TMatcher(TFsm("qw", TFsm::TOptions().SetSurround(true))).Match("aqwb").Final()); + UNIT_ASSERT(!TMatcher(TFsm("qw", TFsm::TOptions().SetSurround(false))).Match("aqwb").Final()); + } + Y_UNIT_TEST(Boundaries) { UNIT_ASSERT(!TMatcher(TFsm("qwb$", TFsm::TOptions().SetSurround(true))).Match("aqwb").Final()); UNIT_ASSERT(!TMatcher(TFsm("^aqw", TFsm::TOptions().SetSurround(true))).Match("aqwb").Final()); @@ -29,13 +29,13 @@ Y_UNIT_TEST_SUITE(TRegExp) { .Match(TStringBuf("q"), false, false) .Match(TStringBuf("w"), false, false) .Match(TStringBuf("b"), false, true) - .Final()); + .Final()); } Y_UNIT_TEST(Case) { - UNIT_ASSERT(TMatcher(TFsm("qw", TFsm::TOptions().SetCaseInsensitive(true))).Match("Qw").Final()); - UNIT_ASSERT(!TMatcher(TFsm("qw", TFsm::TOptions().SetCaseInsensitive(false))).Match("Qw").Final()); - } + UNIT_ASSERT(TMatcher(TFsm("qw", TFsm::TOptions().SetCaseInsensitive(true))).Match("Qw").Final()); + UNIT_ASSERT(!TMatcher(TFsm("qw", TFsm::TOptions().SetCaseInsensitive(false))).Match("Qw").Final()); + } Y_UNIT_TEST(UnicodeCase) { UNIT_ASSERT(TMatcher(TFsm("\\x{61}\\x{62}", TFsm::TOptions().SetCaseInsensitive(true))).Match("Ab").Final()); @@ -114,7 +114,7 @@ Y_UNIT_TEST_SUITE(TRegExp) { Y_UNIT_TEST(Capture3) { TCapturingFsm fsm("http://vk(ontakte[.]ru|[.]com)/id(\\d+)([^0-9]|$)", - TFsm::TOptions().SetCapture(2)); + TFsm::TOptions().SetCapture(2)); TSearcher searcher(fsm); searcher.Search("http://vkontakte.ru/id100500"); @@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(TRegExp) { Y_UNIT_TEST(Capture4) { TCapturingFsm fsm("Здравствуйте, ((\\s|\\w|[()]|-)+)!", - TFsm::TOptions().SetCharset(CODES_UTF8)); + TFsm::TOptions().SetCharset(CODES_UTF8)); TSearcher searcher(fsm); searcher.Search(" Здравствуйте, Уважаемый (-ая)! "); @@ -315,4 +315,4 @@ Y_UNIT_TEST_SUITE(TRegExp) { UNIT_ASSERT_VALUES_EQUAL(Pcre2Pire("(?:(?P<field1>)(?P<field2>))"), ""); UNIT_ASSERT_VALUES_EQUAL(Pcre2Pire("(?:(?:fake))"), "((fake))"); } -} +} diff --git a/library/cpp/regex/pire/ut/ya.make b/library/cpp/regex/pire/ut/ya.make index 8776695f40..0277d88f8c 100644 --- a/library/cpp/regex/pire/ut/ya.make +++ b/library/cpp/regex/pire/ut/ya.make @@ -6,10 +6,10 @@ OWNER( g:util davenger ) - + SET(PIRETESTSDIR contrib/libs/pire/ut) -CFLAGS(-DPIRE_NO_CONFIG) +CFLAGS(-DPIRE_NO_CONFIG) PEERDIR( library/cpp/regex/pire @@ -18,11 +18,11 @@ PEERDIR( SRCDIR( ${PIRETESTSDIR} ) - -ADDINCL( - contrib/libs/pire/pire - contrib/libs/pire/ut -) + +ADDINCL( + contrib/libs/pire/pire + contrib/libs/pire/ut +) SRCS( pire_ut.cpp diff --git a/library/cpp/regex/pire/ya.make b/library/cpp/regex/pire/ya.make index c857e6d18b..7d14c3b043 100644 --- a/library/cpp/regex/pire/ya.make +++ b/library/cpp/regex/pire/ya.make @@ -6,7 +6,7 @@ OWNER( davenger pg ) - + CFLAGS(-DPIRE_NO_CONFIG) SRCDIR(contrib/libs/pire/pire) diff --git a/library/cpp/regex/ya.make b/library/cpp/regex/ya.make index 15b0d1aeda..71fc9a6a43 100644 --- a/library/cpp/regex/ya.make +++ b/library/cpp/regex/ya.make @@ -1,14 +1,14 @@ -RECURSE( - glob +RECURSE( + glob hyperscan hyperscan/ut - libregex - pcre - pire - pire/inline - pire/ut + libregex + pcre + pire + pire/inline + pire/ut pire2hyperscan pire2hyperscan/ut regexp_classifier regexp_classifier/ut -) +) diff --git a/library/cpp/resource/registry.cpp b/library/cpp/resource/registry.cpp index 66001c4769..9ff8f09bb4 100644 --- a/library/cpp/resource/registry.cpp +++ b/library/cpp/resource/registry.cpp @@ -1,25 +1,25 @@ -#include "registry.h" - +#include "registry.h" + #include <library/cpp/blockcodecs/codecs.h> - -#include <util/system/yassert.h> -#include <util/generic/hash.h> -#include <util/generic/deque.h> -#include <util/generic/singleton.h> + +#include <util/system/yassert.h> +#include <util/generic/hash.h> +#include <util/generic/deque.h> +#include <util/generic/singleton.h> #include <util/system/env.h> - -using namespace NResource; -using namespace NBlockCodecs; - -namespace { + +using namespace NResource; +using namespace NBlockCodecs; + +namespace { inline const ICodec* GetCodec() noexcept { - static const ICodec* ret = Codec("zstd08_5"); - - return ret; - } - + static const ICodec* ret = Codec("zstd08_5"); + + return ret; + } + typedef std::pair<TStringBuf, TStringBuf> TDescriptor; - + struct TStore: public IStore, public THashMap<TStringBuf, TDescriptor*> { void Store(const TStringBuf key, const TStringBuf data) override { if (contains(key)) { @@ -41,16 +41,16 @@ namespace { TString{key}.Quote().c_str(), vsize, dsize); } } - } else { - D_.push_back(TDescriptor(key, data)); - (*this)[key] = &D_.back(); - } - + } else { + D_.push_back(TDescriptor(key, data)); + (*this)[key] = &D_.back(); + } + Y_VERIFY(size() == Count(), "size mismatch"); - } - + } + bool FindExact(const TStringBuf key, TString* out) const override { - if (TDescriptor* const* res = FindPtr(key)) { + if (TDescriptor* const* res = FindPtr(key)) { // temporary // https://st.yandex-team.ru/DEVTOOLS-3985 try { @@ -61,13 +61,13 @@ namespace { } throw e; } - - return true; - } - - return false; - } - + + return true; + } + + return false; + } + void FindMatch(const TStringBuf subkey, IMatch& cb) const override { for (const auto& it : *this) { if (it.first.StartsWith(subkey)) { @@ -83,31 +83,31 @@ namespace { } throw e; } - } - } - } - + } + } + } + size_t Count() const noexcept override { - return D_.size(); - } - + return D_.size(); + } + TStringBuf KeyByIndex(size_t idx) const override { - return D_.at(idx).first; - } - + return D_.at(idx).first; + } + typedef TDeque<TDescriptor> TDescriptors; - TDescriptors D_; - }; -} - + TDescriptors D_; + }; +} + TString NResource::Compress(const TStringBuf data) { - return GetCodec()->Encode(data); -} - + return GetCodec()->Encode(data); +} + TString NResource::Decompress(const TStringBuf data) { - return GetCodec()->Decode(data); -} - -IStore* NResource::CommonStore() { - return SingletonWithPriority<TStore, 0>(); -} + return GetCodec()->Decode(data); +} + +IStore* NResource::CommonStore() { + return SingletonWithPriority<TStore, 0>(); +} diff --git a/library/cpp/resource/registry.h b/library/cpp/resource/registry.h index fe67702cbc..a15e000e0a 100644 --- a/library/cpp/resource/registry.h +++ b/library/cpp/resource/registry.h @@ -1,35 +1,35 @@ -#pragma once - +#pragma once + #include <util/generic/string.h> -#include <util/generic/strbuf.h> - -#include "resource.h" - -namespace NResource { +#include <util/generic/strbuf.h> + +#include "resource.h" + +namespace NResource { TString Compress(const TStringBuf data); TString Decompress(const TStringBuf data); - - class IMatch { - public: - virtual void OnMatch(const TResource& res) = 0; - }; - - class IStore { - public: + + class IMatch { + public: + virtual void OnMatch(const TResource& res) = 0; + }; + + class IStore { + public: virtual void Store(const TStringBuf key, const TStringBuf data) = 0; virtual bool FindExact(const TStringBuf key, TString* out) const = 0; virtual void FindMatch(const TStringBuf subkey, IMatch& cb) const = 0; virtual size_t Count() const noexcept = 0; - virtual TStringBuf KeyByIndex(size_t idx) const = 0; + virtual TStringBuf KeyByIndex(size_t idx) const = 0; virtual ~IStore() { } - }; - - IStore* CommonStore(); - - struct TRegHelper { + }; + + IStore* CommonStore(); + + struct TRegHelper { inline TRegHelper(const TStringBuf key, const TStringBuf data) { - CommonStore()->Store(key, data); - } - }; -} + CommonStore()->Store(key, data); + } + }; +} diff --git a/library/cpp/resource/resource.cpp b/library/cpp/resource/resource.cpp index cc20f847a5..a8c2443662 100644 --- a/library/cpp/resource/resource.cpp +++ b/library/cpp/resource/resource.cpp @@ -1,52 +1,52 @@ +#include "resource.h" #include "resource.h" -#include "resource.h" -#include "registry.h" - -#include <util/generic/yexception.h> +#include "registry.h" + +#include <util/generic/yexception.h> #include <util/generic/xrange.h> - -using namespace NResource; - + +using namespace NResource; + bool NResource::FindExact(const TStringBuf key, TString* out) { - return CommonStore()->FindExact(key, out); -} - + return CommonStore()->FindExact(key, out); +} + void NResource::FindMatch(const TStringBuf subkey, TResources* out) { - struct TMatch: public IMatch { - inline TMatch(TResources* r) - : R(r) - { - } - + struct TMatch: public IMatch { + inline TMatch(TResources* r) + : R(r) + { + } + void OnMatch(const TResource& res) override { - R->push_back(res); - } - - TResources* R; - }; - - TMatch m(out); - - CommonStore()->FindMatch(subkey, m); -} - + R->push_back(res); + } + + TResources* R; + }; + + TMatch m(out); + + CommonStore()->FindMatch(subkey, m); +} + TString NResource::Find(const TStringBuf key) { TString ret; - - if (FindExact(key, &ret)) { - return ret; - } - - ythrow yexception() << "can not find resource with path " << key; -} - + + if (FindExact(key, &ret)) { + return ret; + } + + ythrow yexception() << "can not find resource with path " << key; +} + size_t NResource::Count() noexcept { - return CommonStore()->Count(); -} - -TStringBuf NResource::KeyByIndex(size_t idx) { - return CommonStore()->KeyByIndex(idx); -} + return CommonStore()->Count(); +} + +TStringBuf NResource::KeyByIndex(size_t idx) { + return CommonStore()->KeyByIndex(idx); +} TVector<TStringBuf> NResource::ListAllKeys() { TVector<TStringBuf> res(Reserve(NResource::Count())); diff --git a/library/cpp/resource/resource.h b/library/cpp/resource/resource.h index 42dd0f1891..18aa5bd6b2 100644 --- a/library/cpp/resource/resource.h +++ b/library/cpp/resource/resource.h @@ -1,22 +1,22 @@ -#pragma once - +#pragma once + #include <util/generic/string.h> -#include <util/generic/strbuf.h> -#include <util/generic/vector.h> - -namespace NResource { - struct TResource { - TStringBuf Key; +#include <util/generic/strbuf.h> +#include <util/generic/vector.h> + +namespace NResource { + struct TResource { + TStringBuf Key; TString Data; - }; - + }; + typedef TVector<TResource> TResources; - + TString Find(const TStringBuf key); bool FindExact(const TStringBuf key, TString* out); - //perform full scan for now + //perform full scan for now void FindMatch(const TStringBuf subkey, TResources* out); size_t Count() noexcept; - TStringBuf KeyByIndex(size_t idx); + TStringBuf KeyByIndex(size_t idx); TVector<TStringBuf> ListAllKeys(); -} +} diff --git a/library/cpp/resource/ut/lib/data b/library/cpp/resource/ut/lib/data index 50e37d5cac..ef4df624f3 100644 --- a/library/cpp/resource/ut/lib/data +++ b/library/cpp/resource/ut/lib/data @@ -1 +1 @@ -na gorshke sidel korol +na gorshke sidel korol diff --git a/library/cpp/resource/ut/lib/ya.make b/library/cpp/resource/ut/lib/ya.make index 48b92b9c8f..e361cc2c41 100644 --- a/library/cpp/resource/ut/lib/ya.make +++ b/library/cpp/resource/ut/lib/ya.make @@ -1,9 +1,9 @@ -LIBRARY() - -OWNER(pg) - -RESOURCE( - data /x -) - -END() +LIBRARY() + +OWNER(pg) + +RESOURCE( + data /x +) + +END() diff --git a/library/cpp/resource/ut/resource_ut.cpp b/library/cpp/resource/ut/resource_ut.cpp index b6fa8e4df3..7a4f622e15 100644 --- a/library/cpp/resource/ut/resource_ut.cpp +++ b/library/cpp/resource/ut/resource_ut.cpp @@ -1,8 +1,8 @@ #include <library/cpp/resource/resource.h> #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TestResource) { Y_UNIT_TEST(Test1) { - UNIT_ASSERT_VALUES_EQUAL(NResource::Find("/x"), "na gorshke sidel korol\n"); - } -} + UNIT_ASSERT_VALUES_EQUAL(NResource::Find("/x"), "na gorshke sidel korol\n"); + } +} diff --git a/library/cpp/resource/ut/ya.make b/library/cpp/resource/ut/ya.make index d1ac9ed2ed..52ea5b4092 100644 --- a/library/cpp/resource/ut/ya.make +++ b/library/cpp/resource/ut/ya.make @@ -1,9 +1,9 @@ UNITTEST_FOR(library/cpp/resource/ut/lib) - -OWNER(pg) - -SRCS( - resource_ut.cpp -) - -END() + +OWNER(pg) + +SRCS( + resource_ut.cpp +) + +END() diff --git a/library/cpp/resource/ya.make b/library/cpp/resource/ya.make index 9c06df514f..fa85892cc6 100644 --- a/library/cpp/resource/ya.make +++ b/library/cpp/resource/ya.make @@ -1,15 +1,15 @@ -LIBRARY() - -OWNER(pg) - -PEERDIR( +LIBRARY() + +OWNER(pg) + +PEERDIR( library/cpp/blockcodecs/core library/cpp/blockcodecs/codecs/zstd -) - -SRCS( - registry.cpp - resource.cpp -) - -END() +) + +SRCS( + registry.cpp + resource.cpp +) + +END() diff --git a/library/cpp/retry/retry.h b/library/cpp/retry/retry.h index c47ff5070f..36a2585a73 100644 --- a/library/cpp/retry/retry.h +++ b/library/cpp/retry/retry.h @@ -33,7 +33,7 @@ struct TRetryOptions { { } - TRetryOptions& WithCount(ui32 retryCount) { + TRetryOptions& WithCount(ui32 retryCount) { RetryCount = retryCount; return *this; } @@ -65,7 +65,7 @@ struct TRetryOptions { // for compatibility attempt == 0 by default TDuration GetTimeToSleep(ui32 attempt = 0) const { - return SleepDuration + NRetryPrivate::AddRandomDelta(SleepRandomDelta) + NRetryPrivate::AddIncrement(attempt, SleepIncrement) + NRetryPrivate::AddExponentialMultiplier(attempt, SleepExponentialMultiplier); + return SleepDuration + NRetryPrivate::AddRandomDelta(SleepRandomDelta) + NRetryPrivate::AddIncrement(attempt, SleepIncrement) + NRetryPrivate::AddExponentialMultiplier(attempt, SleepExponentialMultiplier); } static TRetryOptions Count(ui32 retryCount) { diff --git a/library/cpp/retry/retry_ut.cpp b/library/cpp/retry/retry_ut.cpp index 92153e987e..bf9de73493 100644 --- a/library/cpp/retry/retry_ut.cpp +++ b/library/cpp/retry/retry_ut.cpp @@ -11,7 +11,7 @@ namespace { } return 42; } - + private: ui32 attempt = 0; }; @@ -21,7 +21,7 @@ namespace { bool operator()() { return (attempt++ == 1); } - + private: ui32 attempt = 0; }; diff --git a/library/cpp/retry/utils.h b/library/cpp/retry/utils.h index a8fd3d1a89..b21ba5977f 100644 --- a/library/cpp/retry/utils.h +++ b/library/cpp/retry/utils.h @@ -7,4 +7,4 @@ namespace NRetryPrivate { TDuration AddIncrement(ui32 attempt, TDuration increment); TDuration AddExponentialMultiplier(ui32 attempt, TDuration exponentialMultiplier); -} +} diff --git a/library/cpp/scheme/domscheme_traits.h b/library/cpp/scheme/domscheme_traits.h index a11c4dd444..29c0bf99a9 100644 --- a/library/cpp/scheme/domscheme_traits.h +++ b/library/cpp/scheme/domscheme_traits.h @@ -1,6 +1,6 @@ #pragma once -#include "scheme.h" +#include "scheme.h" #include <util/string/cast.h> struct TSchemeTraits { @@ -135,28 +135,28 @@ struct TSchemeTraits { return v->IsTrue() || v->IsExplicitFalse(); } -#define INTEGER_OPS_EX(type, min, max, isUnsigned) \ - static inline void Get(TConstValueRef v, type def, type& i) { \ - if (isUnsigned) { \ - i = v->IsNumber() && v->GetIntNumber() >= 0 ? v->GetIntNumber() : def; \ - } else { \ - i = v->IsNumber() ? v->GetIntNumber() : def; \ - } \ - } \ - static inline void Get(TConstValueRef v, type& i) { \ - if (isUnsigned) { \ - i = Max<i64>(0, v->GetIntNumber()); \ - } else { \ - i = v->GetIntNumber(); \ - } \ - } \ - static inline bool IsValidPrimitive(const type&, TConstValueRef v) { \ - return v->IsIntNumber() && \ - v->GetIntNumber() >= min && \ - v->GetIntNumber() <= max; \ - } \ - static inline void Set(TValueRef v, type i) { \ - v->SetIntNumber(i); \ +#define INTEGER_OPS_EX(type, min, max, isUnsigned) \ + static inline void Get(TConstValueRef v, type def, type& i) { \ + if (isUnsigned) { \ + i = v->IsNumber() && v->GetIntNumber() >= 0 ? v->GetIntNumber() : def; \ + } else { \ + i = v->IsNumber() ? v->GetIntNumber() : def; \ + } \ + } \ + static inline void Get(TConstValueRef v, type& i) { \ + if (isUnsigned) { \ + i = Max<i64>(0, v->GetIntNumber()); \ + } else { \ + i = v->GetIntNumber(); \ + } \ + } \ + static inline bool IsValidPrimitive(const type&, TConstValueRef v) { \ + return v->IsIntNumber() && \ + v->GetIntNumber() >= min && \ + v->GetIntNumber() <= max; \ + } \ + static inline void Set(TValueRef v, type i) { \ + v->SetIntNumber(i); \ } #define INTEGER_OPS(type, isUnsigned) INTEGER_OPS_EX(type, Min<type>(), Max<type>(), isUnsigned) diff --git a/library/cpp/scheme/scheme.cpp b/library/cpp/scheme/scheme.cpp index 3efd116d4f..77439119d3 100644 --- a/library/cpp/scheme/scheme.cpp +++ b/library/cpp/scheme/scheme.cpp @@ -156,7 +156,7 @@ namespace NSc { TValue& TValue::DoCopyFromImpl(const TValue& other, NImpl::TSelfLoopContext& otherLoopCtx, - NImpl::TSelfOverrideContext& selfOverrideCtx) { + NImpl::TSelfOverrideContext& selfOverrideCtx) { if (Same(*this, other)) { return *this; } @@ -176,44 +176,44 @@ namespace NSc { } switch (otherCore.ValueType) { - default: - Y_ASSERT(false); + default: + Y_ASSERT(false); [[fallthrough]]; - case EType::Null: - break; - case EType::Bool: - selfCore.SetBool(otherCore.IntNumber); - break; - case EType::IntNumber: - selfCore.SetIntNumber(otherCore.IntNumber); - break; - case EType::FloatNumber: - selfCore.SetNumber(otherCore.FloatNumber); - break; - case EType::String: + case EType::Null: + break; + case EType::Bool: + selfCore.SetBool(otherCore.IntNumber); + break; + case EType::IntNumber: + selfCore.SetIntNumber(otherCore.IntNumber); + break; + case EType::FloatNumber: + selfCore.SetNumber(otherCore.FloatNumber); + break; + case EType::String: if (selfCore.Pool.Get() == otherCore.Pool.Get()) { selfCore.SetOwnedString(otherCore.String); } else { selfCore.SetString(otherCore.String); } - break; - case EType::Array: - selfCore.SetArray(); - for (const TValue& e : otherCore.GetArray()) { + break; + case EType::Array: + selfCore.SetArray(); + for (const TValue& e : otherCore.GetArray()) { selfCore.Push().DoCopyFromImpl(e, otherLoopCtx, selfOverrideCtx); - } - break; - case EType::Dict: { + } + break; + case EType::Dict: { TCorePtr tmp = NewCore(selfCore.Pool); auto& tmpCore = *tmp; tmpCore.SetDict(); - const TDict& d = otherCore.GetDict(); + const TDict& d = otherCore.GetDict(); tmpCore.Dict.reserve(d.size()); - for (const TDict::value_type& e : d) { + for (const TDict::value_type& e : d) { tmpCore.Add(e.first).DoCopyFromImpl(e.second, otherLoopCtx, selfOverrideCtx); - } + } TheCore = std::move(tmp); - break; + break; } } @@ -283,7 +283,7 @@ namespace NSc { if (itb == dictB.end() || !Equal(ita.second, itb->second)) { return false; - } + } } } @@ -301,7 +301,7 @@ namespace NSc { TValue& TValue::DoMergeImpl(const TValue& delta, bool lowPriorityDelta, NImpl::TSelfLoopContext& otherLoopCtx, - NImpl::TSelfOverrideContext& selfOverrideGuard) { + NImpl::TSelfOverrideContext& selfOverrideGuard) { if (Same(*this, delta)) { return *this; } @@ -341,48 +341,48 @@ namespace NSc { const TScCore& core = Core(); switch (core.ValueType) { - default: - case EType::Null: + default: + case EType::Null: return NJson::TJsonValue(NJson::JSON_NULL); - case EType::Bool: - return NJson::TJsonValue(core.GetBool()); - case EType::IntNumber: - return NJson::TJsonValue(core.GetIntNumber()); - case EType::FloatNumber: - return NJson::TJsonValue(core.GetNumber()); - case EType::String: - return NJson::TJsonValue(core.String); - case EType::Array: { - NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); - - if (!loopGuard.Ok) { - return NJson::TJsonValue(NJson::JSON_NULL); - } - - NJson::TJsonValue result(NJson::JSON_ARRAY); - const TArray& arr = core.Array; - - for (const auto& item : arr) { - result.AppendValue(NJson::TJsonValue::UNDEFINED) = item.ToJsonValueImpl(loopCtx); - } - - return result; + case EType::Bool: + return NJson::TJsonValue(core.GetBool()); + case EType::IntNumber: + return NJson::TJsonValue(core.GetIntNumber()); + case EType::FloatNumber: + return NJson::TJsonValue(core.GetNumber()); + case EType::String: + return NJson::TJsonValue(core.String); + case EType::Array: { + NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); + + if (!loopGuard.Ok) { + return NJson::TJsonValue(NJson::JSON_NULL); + } + + NJson::TJsonValue result(NJson::JSON_ARRAY); + const TArray& arr = core.Array; + + for (const auto& item : arr) { + result.AppendValue(NJson::TJsonValue::UNDEFINED) = item.ToJsonValueImpl(loopCtx); + } + + return result; } - case EType::Dict: { - NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); - - if (!loopGuard.Ok) { - return NJson::TJsonValue(NJson::JSON_NULL); - } - - NJson::TJsonValue result(NJson::JSON_MAP); - const TDict& dict = core.Dict; - - for (const auto& item : dict) { - result.InsertValue(item.first, NJson::TJsonValue::UNDEFINED) = item.second.ToJsonValueImpl(loopCtx); - } - - return result; + case EType::Dict: { + NImpl::TSelfLoopContext::TGuard loopGuard(loopCtx, core); + + if (!loopGuard.Ok) { + return NJson::TJsonValue(NJson::JSON_NULL); + } + + NJson::TJsonValue result(NJson::JSON_MAP); + const TDict& dict = core.Dict; + + for (const auto& item : dict) { + result.InsertValue(item.first, NJson::TJsonValue::UNDEFINED) = item.second.ToJsonValueImpl(loopCtx); + } + + return result; } } } @@ -398,38 +398,38 @@ namespace NSc { core.SetNull(); switch (val.GetType()) { - default: - case NJson::JSON_UNDEFINED: - case NJson::JSON_NULL: - break; - case NJson::JSON_BOOLEAN: - core.SetBool(val.GetBoolean()); - break; - case NJson::JSON_INTEGER: - core.SetIntNumber(val.GetInteger()); - break; - case NJson::JSON_UINTEGER: - core.SetIntNumber(val.GetUInteger()); - break; - case NJson::JSON_DOUBLE: - core.SetNumber(val.GetDouble()); - break; - case NJson::JSON_STRING: + default: + case NJson::JSON_UNDEFINED: + case NJson::JSON_NULL: + break; + case NJson::JSON_BOOLEAN: + core.SetBool(val.GetBoolean()); + break; + case NJson::JSON_INTEGER: + core.SetIntNumber(val.GetInteger()); + break; + case NJson::JSON_UINTEGER: + core.SetIntNumber(val.GetUInteger()); + break; + case NJson::JSON_DOUBLE: + core.SetNumber(val.GetDouble()); + break; + case NJson::JSON_STRING: core.SetString(val.GetString()); - break; - case NJson::JSON_ARRAY: { + break; + case NJson::JSON_ARRAY: { core.SetArray(); - for (const auto& item : val.GetArray()) { + for (const auto& item : val.GetArray()) { FromJsonValue(core.Push(), item); - } - break; + } + break; } - case NJson::JSON_MAP: { + case NJson::JSON_MAP: { core.SetDict(); - for (const auto& item : val.GetMap()) { + for (const auto& item : val.GetMap()) { FromJsonValue(core.Add(item.first), item.second); - } - break; + } + break; } } @@ -449,7 +449,7 @@ namespace NSc { return Default<TDefaults>().Core.Array; } - const TDict& TValue::DefaultDict() { + const TDict& TValue::DefaultDict() { return Default<TDefaults>().Core.Dict; } @@ -506,69 +506,69 @@ namespace NSc { } namespace NPrivate { - int CompareStr(const NSc::TValue& a, TStringBuf b) { + int CompareStr(const NSc::TValue& a, TStringBuf b) { return a.GetString().compare(b); - } + } - int CompareInt(const NSc::TValue& a, i64 r) { - i64 l = a.GetIntNumber(); - return l < r ? -1 : l > r ? 1 : 0; - } + int CompareInt(const NSc::TValue& a, i64 r) { + i64 l = a.GetIntNumber(); + return l < r ? -1 : l > r ? 1 : 0; + } - int CompareFloat(const NSc::TValue& a, double r) { - double l = a.GetNumber(); - return l < r ? -1 : l > r ? 1 : 0; - } + int CompareFloat(const NSc::TValue& a, double r) { + double l = a.GetNumber(); + return l < r ? -1 : l > r ? 1 : 0; + } } - bool operator==(const NSc::TValue& a, const NSc::TValue& b) { + bool operator==(const NSc::TValue& a, const NSc::TValue& b) { return NSc::TValue::Equal(a, b); } - bool operator!=(const NSc::TValue& a, const NSc::TValue& b) { + bool operator!=(const NSc::TValue& a, const NSc::TValue& b) { return !NSc::TValue::Equal(a, b); } #define LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(T, Impl) \ - bool operator==(const NSc::TValue& a, T b) { \ - return NPrivate::Impl(a, b) == 0; \ - } \ - bool operator==(T b, const NSc::TValue& a) { \ - return NPrivate::Impl(a, b) == 0; \ - } \ - bool operator!=(const NSc::TValue& a, T b) { \ - return NPrivate::Impl(a, b) != 0; \ - } \ - bool operator!=(T b, const NSc::TValue& a) { \ - return NPrivate::Impl(a, b) != 0; \ - } \ - bool operator<=(const NSc::TValue& a, T b) { \ - return NPrivate::Impl(a, b) <= 0; \ - } \ - bool operator<=(T b, const NSc::TValue& a) { \ - return NPrivate::Impl(a, b) >= 0; \ - } \ - bool operator>=(const NSc::TValue& a, T b) { \ - return NPrivate::Impl(a, b) >= 0; \ - } \ - bool operator>=(T b, const NSc::TValue& a) { \ - return NPrivate::Impl(a, b) <= 0; \ - } \ - bool operator<(const NSc::TValue& a, T b) { \ - return NPrivate::Impl(a, b) < 0; \ - } \ - bool operator<(T b, const NSc::TValue& a) { \ - return NPrivate::Impl(a, b) > 0; \ - } \ - bool operator>(const NSc::TValue& a, T b) { \ - return NPrivate::Impl(a, b) > 0; \ - } \ - bool operator>(T b, const NSc::TValue& a) { \ - return NPrivate::Impl(a, b) < 0; \ - } - -#define LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(T) \ + bool operator==(const NSc::TValue& a, T b) { \ + return NPrivate::Impl(a, b) == 0; \ + } \ + bool operator==(T b, const NSc::TValue& a) { \ + return NPrivate::Impl(a, b) == 0; \ + } \ + bool operator!=(const NSc::TValue& a, T b) { \ + return NPrivate::Impl(a, b) != 0; \ + } \ + bool operator!=(T b, const NSc::TValue& a) { \ + return NPrivate::Impl(a, b) != 0; \ + } \ + bool operator<=(const NSc::TValue& a, T b) { \ + return NPrivate::Impl(a, b) <= 0; \ + } \ + bool operator<=(T b, const NSc::TValue& a) { \ + return NPrivate::Impl(a, b) >= 0; \ + } \ + bool operator>=(const NSc::TValue& a, T b) { \ + return NPrivate::Impl(a, b) >= 0; \ + } \ + bool operator>=(T b, const NSc::TValue& a) { \ + return NPrivate::Impl(a, b) <= 0; \ + } \ + bool operator<(const NSc::TValue& a, T b) { \ + return NPrivate::Impl(a, b) < 0; \ + } \ + bool operator<(T b, const NSc::TValue& a) { \ + return NPrivate::Impl(a, b) > 0; \ + } \ + bool operator>(const NSc::TValue& a, T b) { \ + return NPrivate::Impl(a, b) > 0; \ + } \ + bool operator>(T b, const NSc::TValue& a) { \ + return NPrivate::Impl(a, b) < 0; \ + } + +#define LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS_IMPL(T) \ LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(signed T, CompareInt) \ LIBRARY_SCHEME_DECLARE_TVALUE_OPS_IMPL(unsigned T, CompareInt) diff --git a/library/cpp/scheme/scheme.h b/library/cpp/scheme/scheme.h index 3d7c59f3c9..ca69752b7d 100644 --- a/library/cpp/scheme/scheme.h +++ b/library/cpp/scheme/scheme.h @@ -9,20 +9,20 @@ namespace NSc { #ifdef _MSC_VER -#pragma warning(disable : 4521 4522) +#pragma warning(disable : 4521 4522) #endif // todo: try to remove some rarely used methods class TValue { public: enum class EType { - Null = 0 /* "Null" */, - Bool /* "Bool" */, - IntNumber /* "Int" */, - FloatNumber /* "Float" */, - String /* "String" */, - Array /* "Array" */, - Dict /* "Dict" */ + Null = 0 /* "Null" */, + Bool /* "Bool" */, + IntNumber /* "Int" */, + FloatNumber /* "Float" */, + String /* "String" */, + Array /* "Array" */, + Dict /* "Dict" */ }; struct TScCore; @@ -53,7 +53,7 @@ namespace NSc { inline TValue(long long t); inline TValue(long t); inline TValue(int t); - // inline TValue(bool b); + // inline TValue(bool b); inline TValue(TStringBuf t); inline TValue(const char*); @@ -86,7 +86,7 @@ namespace NSc { inline TValue& operator=(long long t); inline TValue& operator=(long t); inline TValue& operator=(int t); - // inline TValue& operator=(bool t); + // inline TValue& operator=(bool t); inline TValue& operator=(TStringBuf t); inline TValue& operator=(const char* t); @@ -101,14 +101,14 @@ namespace NSc { public: template <class T> // ui16 or TStringBuf - inline TValue& operator[](const T& idx) { - return GetOrAdd(idx); - } + inline TValue& operator[](const T& idx) { + return GetOrAdd(idx); + } template <class T> // ui16 or TStringBuf - inline const TValue& operator[](const T& idx) const { - return Get(idx); - } + inline const TValue& operator[](const T& idx) const { + return Get(idx); + } public: // Data methods /////////////////////////////////////////////////////////// inline EType GetType() const; @@ -117,9 +117,9 @@ namespace NSc { inline TValue& SetNull(); // returns self, will set type to Null - TValue& Clear() { - return ClearArray().ClearDict().SetNull(); - } + TValue& Clear() { + return ClearArray().ClearDict().SetNull(); + } public: // Number methods ///////////////////////////////////////////////////////// // Bool, IntNumber and FloatNumber are all compatible. @@ -179,7 +179,7 @@ namespace NSc { TString ForceString(const TString& deflt = TString()) const; // Best-effort cast to TString (will do ToString for numeric types) // todo: remove - inline bool StringEmpty() const; + inline bool StringEmpty() const; inline size_t StringSize() const; public: // Array methods ////////////////////////////////////////////////////////// @@ -202,26 +202,26 @@ namespace NSc { return Push() = std::forward<T>(t); } // returns new child - TValue& Insert(ui16 idx) { - return InsertUnsafe(idx); - } // creates missing values, returns new child + TValue& Insert(ui16 idx) { + return InsertUnsafe(idx); + } // creates missing values, returns new child template <class T> TValue& Insert(ui16 idx, T&& v) { return InsertUnsafe(idx, std::forward<T>(v)); - } // creates missing values, returns new child + } // creates missing values, returns new child template <class TIt> inline TValue& AppendAll(TIt begin, TIt end); // Append(vec.begin(), vec.end()) template <class TColl> - inline TValue& AppendAll(TColl&& coll); // Append(vec) + inline TValue& AppendAll(TColl&& coll); // Append(vec) inline TValue& AppendAll(std::initializer_list<TValue> coll); - TValue& GetOrAdd(ui16 idx) { - return GetOrAddUnsafe(idx); - } // creates missing values, returns new child + TValue& GetOrAdd(ui16 idx) { + return GetOrAddUnsafe(idx); + } // creates missing values, returns new child inline TValue& InsertUnsafe(size_t idx); // creates missing values, returns new child @@ -230,17 +230,17 @@ namespace NSc { return InsertUnsafe(idx) = std::forward<T>(t); } // creates missing values, returns new child - inline TValue& GetOrAddUnsafe(size_t idx); // creates missing values, returns new child + inline TValue& GetOrAddUnsafe(size_t idx); // creates missing values, returns new child - inline TValue Pop(); // returns popped value + inline TValue Pop(); // returns popped value inline TValue Delete(size_t idx); // returns deleted value if it existed, NSc::Null() otherwise - inline TValue& Front() { - return GetOrAdd(0); - } // creates missing value, returns child - inline const TValue& Front() const { - return Get(0); - } // returns child or default + inline TValue& Front() { + return GetOrAdd(0); + } // creates missing value, returns child + inline const TValue& Front() const { + return Get(0); + } // returns child or default inline TValue& Back(); // creates missing value, returns child inline const TValue& Back() const; // returns child or default @@ -262,14 +262,14 @@ namespace NSc { inline const TValue& Get(TStringBuf idx) const; inline TValue* GetNoAdd(TStringBuf idx); // returns link to existing child or nullptr - TValue& Add(TStringBuf idx) { - return GetOrAdd(idx); - } + TValue& Add(TStringBuf idx) { + return GetOrAdd(idx); + } template <class T> TValue& Add(TStringBuf idx, T&& t) { return Add(idx) = std::forward<T>(t); - } + } inline TValue& GetOrAdd(TStringBuf idx); // creates missing value, returns child @@ -291,9 +291,9 @@ namespace NSc { static const EJsonOpts JO_SORT_KEYS = TJsonOpts::JO_SORT_KEYS; static const EJsonOpts JO_SKIP_UNSAFE = TJsonOpts::JO_SKIP_UNSAFE; // skip non-utf8 strings static const EJsonOpts JO_PRETTY = TJsonOpts::JO_PRETTY; - static const EJsonOpts JO_SAFE = TJsonOpts::JO_SAFE; // JO_SORT_KEYS | JO_SKIP_UNSAFE + static const EJsonOpts JO_SAFE = TJsonOpts::JO_SAFE; // JO_SORT_KEYS | JO_SKIP_UNSAFE static const EJsonOpts JO_PARSER_STRICT_WITH_COMMENTS = TJsonOpts::JO_PARSER_STRICT_WITH_COMMENTS; // strict json + strict utf8 - static const EJsonOpts JO_PARSER_STRICT = TJsonOpts::JO_PARSER_STRICT; // strict json + strict utf8 + comments are disallowed + static const EJsonOpts JO_PARSER_STRICT = TJsonOpts::JO_PARSER_STRICT; // strict json + strict utf8 + comments are disallowed static const EJsonOpts JO_PARSER_DISALLOW_DUPLICATE_KEYS = TJsonOpts::JO_PARSER_DISALLOW_DUPLICATE_KEYS; static TValue FromJson(TStringBuf, const TJsonOpts& = TJsonOpts()); @@ -321,7 +321,7 @@ namespace NSc { static TJsonOpts MakeOptsPrettyForSerializer(TJsonOpts = TJsonOpts()); public: // Merge methods //////////////////////////////////////////////// - /* + /* * LHS.MergeUpdate(RHS): * 1. Dict <- Dict: * - Copy all nonconflicting key-value pairs from RHS to LHS. @@ -347,7 +347,7 @@ namespace NSc { static bool MergeUpdateJson(TValue&, TStringBuf json); // returns true unless failed to parse the json static bool ReverseMergeJson(TValue&, TStringBuf json); // returns true unless failed to parse the json - TValue& MergeUpdate(const TValue& delta); // return self + TValue& MergeUpdate(const TValue& delta); // return self TValue& ReverseMerge(const TValue& delta); // return self public: // Path methods ///////////////////////////////////////////////////////// @@ -356,9 +356,9 @@ namespace NSc { static TString EscapeForPath(TStringBuf rawKey); // converts a raw dict key into a valid token for a selector path - static bool PathValid(TStringBuf path); // returns true if the path is syntactically valid + static bool PathValid(TStringBuf path); // returns true if the path is syntactically valid - bool PathExists(TStringBuf path) const; // returns true if the path is syntactically valid and the target value exists + bool PathExists(TStringBuf path) const; // returns true if the path is syntactically valid and the target value exists const TValue& TrySelect(TStringBuf path) const; // returns the target value // if the path is syntactically valid and the target value exists @@ -372,14 +372,14 @@ namespace NSc { // if the path is syntactically valid and the target value existed // otherwise returns NSc::Null() - public: // Copy methods ///////////////////////////////////////////////////////// - TValue Clone() const; // returns deep copy of self (on the separate pool) + public: // Copy methods ///////////////////////////////////////////////////////// + TValue Clone() const; // returns deep copy of self (on the separate pool) TValue& CopyFrom(const TValue& other); // deep copy other value into self, returns self TValue& Swap(TValue& v); - static bool Same(const TValue&, const TValue&); // point to the same core - static bool Equal(const TValue&, const TValue&); // recursively equal + static bool Same(const TValue&, const TValue&); // point to the same core + static bool Equal(const TValue&, const TValue&); // recursively equal static bool SamePool(const TValue&, const TValue&); // share arena public: @@ -394,7 +394,7 @@ namespace NSc { static const TScCore& DefaultCore(); static const TArray& DefaultArray(); - static const TDict& DefaultDict(); + static const TDict& DefaultDict(); static const TValue& DefaultValue(); static const TValue& Null() { return DefaultValue(); @@ -434,14 +434,14 @@ namespace NSc { } - class TArray: public TDeque<TValue, TPoolAllocator>, TNonCopyable { - using TParent = TDeque<TValue, TPoolAllocator>; + class TArray: public TDeque<TValue, TPoolAllocator>, TNonCopyable { + using TParent = TDeque<TValue, TPoolAllocator>; public: TArray(TMemoryPool* p) : TParent(p) - { - } + { + } template <class TIt> void AppendAll(TIt begin, TIt end) { @@ -505,7 +505,7 @@ namespace NSc { } const TValue& EnsureIndex(size_t i) const { - return i < size() ? TParent::operator[](i) : TValue::DefaultValue(); + return i < size() ? TParent::operator[](i) : TValue::DefaultValue(); } }; @@ -513,14 +513,14 @@ namespace NSc { // todo: densehashtable // todo: allow insertions // todo: make TDict methods safe - class TDict: public THashMap<TStringBuf, TValue, THash<TStringBuf>, TEqualTo<TStringBuf>, TPoolAllocator>, TNonCopyable { + class TDict: public THashMap<TStringBuf, TValue, THash<TStringBuf>, TEqualTo<TStringBuf>, TPoolAllocator>, TNonCopyable { using TParent = THashMap<TStringBuf, TValue, THash<TStringBuf>, TEqualTo<TStringBuf>, TPoolAllocator>; - + public: TDict(TMemoryPool* p) : TParent(p) - { - } + { + } template <class TStr> const TValue& Get(const TStr& key) const { @@ -534,5 +534,5 @@ namespace NSc { #include "scheme_cast.h" #ifdef _MSC_VER -#pragma warning(default : 4521 4522) +#pragma warning(default : 4521 4522) #endif diff --git a/library/cpp/scheme/scheme_cast.h b/library/cpp/scheme/scheme_cast.h index 00839e8017..a43c4e07d3 100644 --- a/library/cpp/scheme/scheme_cast.h +++ b/library/cpp/scheme/scheme_cast.h @@ -8,28 +8,28 @@ #include <util/string/cast.h> #include <util/generic/yexception.h> -#include "scheme.h" +#include "scheme.h" namespace NJsonConverters { class IJsonSerializable { - public: - virtual NSc::TValue ToTValue() const = 0; - virtual void FromTValue(const NSc::TValue&, const bool validate) = 0; + public: + virtual NSc::TValue ToTValue() const = 0; + virtual void FromTValue(const NSc::TValue&, const bool validate) = 0; - const TString ToJson(const bool sort = false) const { - return ToTValue().ToJson(sort); - }; + const TString ToJson(const bool sort = false) const { + return ToTValue().ToJson(sort); + }; - void FromJson(const TStringBuf& json, const bool validate = false) { - NSc::TValue v = NSc::TValue::FromJson(json); - FromTValue(v, validate); - } + void FromJson(const TStringBuf& json, const bool validate = false) { + NSc::TValue v = NSc::TValue::FromJson(json); + FromTValue(v, validate); + } - virtual ~IJsonSerializable(){}; + virtual ~IJsonSerializable(){}; }; - ////////////////////////////////////////////////////////////////////// - // fwd declarations - ////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////// + // fwd declarations + ////////////////////////////////////////////////////////////////////// //TVector template <typename T, typename A> @@ -67,13 +67,13 @@ namespace NJsonConverters { template <class T1, class T2> void FromTValue(const NSc::TValue& x, std::pair<T1, T2>& out, const bool validate); - ////////////////////////////////////////////////////////////////////// - // simple From, To helpers - ////////////////////////////////////////////////////////////////////// - template <typename T, bool HasSerializer> + ////////////////////////////////////////////////////////////////////// + // simple From, To helpers + ////////////////////////////////////////////////////////////////////// + template <typename T, bool HasSerializer> struct TValueAndStrokaConv {}; - template <typename T> + template <typename T> struct TValueAndStrokaConv<T, 0> { static NSc::TValue ToTValue(const T& x) { return NSc::TValue(x); @@ -92,8 +92,8 @@ namespace NJsonConverters { } }; - template <typename T> - struct TValueAndStrokaConv<T, 1> { + template <typename T> + struct TValueAndStrokaConv<T, 1> { static NSc::TValue ToTValue(const T& x) { return x.ToTValue(); } @@ -113,12 +113,12 @@ namespace NJsonConverters { template <typename T> NSc::TValue ToTValue(const T& x) { - return TValueAndStrokaConv<T, std::is_base_of<IJsonSerializable, T>::value>::ToTValue(x); + return TValueAndStrokaConv<T, std::is_base_of<IJsonSerializable, T>::value>::ToTValue(x); } template <typename T> void FromTValue(const NSc::TValue& x, T& out, const bool validate) { - return TValueAndStrokaConv<T, std::is_base_of<IJsonSerializable, T>::value>::FromTValue(x, out, validate); + return TValueAndStrokaConv<T, std::is_base_of<IJsonSerializable, T>::value>::FromTValue(x, out, validate); } template <typename T> @@ -130,12 +130,12 @@ namespace NJsonConverters { template <typename T> TString ToString(const T& x) { - return TValueAndStrokaConv<T, std::is_base_of<IJsonSerializable, T>::value>::ToString(x); + return TValueAndStrokaConv<T, std::is_base_of<IJsonSerializable, T>::value>::ToString(x); } template <typename T> void FromString(const TStringBuf& str, T& res, const bool validate) { - return TValueAndStrokaConv<T, std::is_base_of<IJsonSerializable, T>::value>::FromString(str, res, validate); + return TValueAndStrokaConv<T, std::is_base_of<IJsonSerializable, T>::value>::FromString(str, res, validate); } template <typename T> @@ -200,9 +200,9 @@ namespace NJsonConverters { } } - ////////////////////////////////////////////////////////////////////// - // TVector - ////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////// + // TVector + ////////////////////////////////////////////////////////////////////// template <typename T, typename A> NSc::TValue ToTValue(const TVector<T, A>& x) { NSc::TValue out; @@ -228,9 +228,9 @@ namespace NJsonConverters { } } - ////////////////////////////////////////////////////////////////////// - // THashMap & TMap - ////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////// + // THashMap & TMap + ////////////////////////////////////////////////////////////////////// template <class Key, class T, class HashFcn, class EqualKey, class Alloc> NSc::TValue ToTValue(const THashMap<Key, T, HashFcn, EqualKey, Alloc>& x) { return NPrivate::ToTValueDict(x); @@ -251,9 +251,9 @@ namespace NJsonConverters { NPrivate::FromTValueDict(x, out, validate); } - ////////////////////////////////////////////////////////////////////// - // THashSet & TSet - ////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////// + // THashSet & TSet + ////////////////////////////////////////////////////////////////////// template <class V, class H, class E, class A> NSc::TValue ToTValue(const THashSet<V, H, E, A>& x) { return NPrivate::ToTValueSet(x); @@ -274,9 +274,9 @@ namespace NJsonConverters { NPrivate::FromTValueSet(x, out, validate); } - ////////////////////////////////////////////////////////////////////// - // std::pair - ////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////// + // std::pair + ////////////////////////////////////////////////////////////////////// template <class T1, class T2> NSc::TValue ToTValue(const std::pair<T1, T2>& x) { NSc::TValue out; @@ -303,9 +303,9 @@ namespace NJsonConverters { } } - ////////////////////////////////////////////////////////////////////// - // global user functions - ////////////////////////////////////////////////////////////////////// + ////////////////////////////////////////////////////////////////////// + // global user functions + ////////////////////////////////////////////////////////////////////// template <typename T> TString ToJson(const T& val, const bool sort = false) { return NJsonConverters::ToTValue(val).ToJson(sort); diff --git a/library/cpp/scheme/scimpl.h b/library/cpp/scheme/scimpl.h index 4f68f16290..fafea0826a 100644 --- a/library/cpp/scheme/scimpl.h +++ b/library/cpp/scheme/scimpl.h @@ -18,8 +18,8 @@ namespace NSc { : Pool(p) , Dict(Pool->Get()) , Array(Pool->Get()) - { - } + { + } bool IsNull() const { return TValue::EType::Null == ValueType; @@ -101,17 +101,17 @@ namespace NSc { double& GetNumberMutable(double defaultnum) { switch (ValueType) { - case TValue::EType::Bool: - SetNumber(bool(IntNumber)); - break; - case TValue::EType::IntNumber: - SetNumber(IntNumber); - break; - case TValue::EType::FloatNumber: - break; - default: - SetNumber(defaultnum); - break; + case TValue::EType::Bool: + SetNumber(bool(IntNumber)); + break; + case TValue::EType::IntNumber: + SetNumber(IntNumber); + break; + case TValue::EType::FloatNumber: + break; + default: + SetNumber(defaultnum); + break; } return FloatNumber; @@ -119,17 +119,17 @@ namespace NSc { i64& GetIntNumberMutable(i64 defaultnum) { switch (ValueType) { - case TValue::EType::Bool: - SetIntNumber(bool(IntNumber)); - break; - case TValue::EType::IntNumber: - break; - case TValue::EType::FloatNumber: - SetIntNumber(FloatNumber); - break; - default: - SetIntNumber(defaultnum); - break; + case TValue::EType::Bool: + SetIntNumber(bool(IntNumber)); + break; + case TValue::EType::IntNumber: + break; + case TValue::EType::FloatNumber: + SetIntNumber(FloatNumber); + break; + default: + SetIntNumber(defaultnum); + break; } return IntNumber; @@ -153,27 +153,27 @@ namespace NSc { double GetNumber(double d = 0) const { switch (ValueType) { - case TValue::EType::Bool: - return (bool)IntNumber; - case TValue::EType::IntNumber: - return IntNumber; - case TValue::EType::FloatNumber: - return FloatNumber; - default: - return d; + case TValue::EType::Bool: + return (bool)IntNumber; + case TValue::EType::IntNumber: + return IntNumber; + case TValue::EType::FloatNumber: + return FloatNumber; + default: + return d; } } i64 GetIntNumber(i64 n = 0) const { switch (ValueType) { - case TValue::EType::Bool: - return (bool)IntNumber; - case TValue::EType::IntNumber: - return IntNumber; - case TValue::EType::FloatNumber: - return FloatNumber; - default: - return n; + case TValue::EType::Bool: + return (bool)IntNumber; + case TValue::EType::IntNumber: + return IntNumber; + case TValue::EType::FloatNumber: + return FloatNumber; + default: + return n; } } @@ -399,14 +399,14 @@ namespace NSc { TValue::TValue(TValue& v) : TheCore(v.TheCore) , CopyOnWrite(v.CopyOnWrite) - { - } + { + } TValue::TValue(const TValue& v) : TheCore(v.TheCore) , CopyOnWrite(true) - { - } + { + } TValue::TValue(TValue&& v) noexcept : TheCore(std::move(v.TheCore)) @@ -609,13 +609,13 @@ namespace NSc { template <class TIt> TValue& TValue::AppendAll(TIt begin, TIt end) { - GetArrayMutable().AppendAll(begin, end); - return *this; + GetArrayMutable().AppendAll(begin, end); + return *this; } template <class TColl> TValue& TValue::AppendAll(TColl&& coll) { - return AppendAll(std::begin(coll), std::end(coll)); + return AppendAll(std::begin(coll), std::end(coll)); } TValue& TValue::AppendAll(std::initializer_list<TValue> coll) { @@ -810,30 +810,30 @@ namespace NSc { int CompareFloat(const NSc::TValue& a, double r); } - bool operator==(const TValue& a, const TValue& b); + bool operator==(const TValue& a, const TValue& b); - bool operator!=(const TValue& a, const TValue& b); + bool operator!=(const TValue& a, const TValue& b); - bool operator<=(const TValue&, const TValue&) = delete; - bool operator>=(const TValue&, const TValue&) = delete; - bool operator<(const TValue&, const TValue&) = delete; - bool operator>(const TValue&, const TValue&) = delete; + bool operator<=(const TValue&, const TValue&) = delete; + bool operator>=(const TValue&, const TValue&) = delete; + bool operator<(const TValue&, const TValue&) = delete; + bool operator>(const TValue&, const TValue&) = delete; #define LIBRARY_SCHEME_DECLARE_TVALUE_OPS(T, Impl) \ - bool operator==(const NSc::TValue& a, T b); \ - bool operator==(T b, const NSc::TValue& a); \ - bool operator!=(const NSc::TValue& a, T b); \ - bool operator!=(T b, const NSc::TValue& a); \ - bool operator<=(const NSc::TValue& a, T b); \ - bool operator<=(T b, const NSc::TValue& a); \ - bool operator>=(const NSc::TValue& a, T b); \ - bool operator>=(T b, const NSc::TValue& a); \ - bool operator<(const NSc::TValue& a, T b); \ - bool operator<(T b, const NSc::TValue& a); \ - bool operator>(const NSc::TValue& a, T b); \ - bool operator>(T b, const NSc::TValue& a); - -#define LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(T) \ + bool operator==(const NSc::TValue& a, T b); \ + bool operator==(T b, const NSc::TValue& a); \ + bool operator!=(const NSc::TValue& a, T b); \ + bool operator!=(T b, const NSc::TValue& a); \ + bool operator<=(const NSc::TValue& a, T b); \ + bool operator<=(T b, const NSc::TValue& a); \ + bool operator>=(const NSc::TValue& a, T b); \ + bool operator>=(T b, const NSc::TValue& a); \ + bool operator<(const NSc::TValue& a, T b); \ + bool operator<(T b, const NSc::TValue& a); \ + bool operator>(const NSc::TValue& a, T b); \ + bool operator>(T b, const NSc::TValue& a); + +#define LIBRARY_SCHEME_DECLARE_TVALUE_INT_OPS(T) \ LIBRARY_SCHEME_DECLARE_TVALUE_OPS(signed T, CompareInt) \ LIBRARY_SCHEME_DECLARE_TVALUE_OPS(unsigned T, CompareInt) diff --git a/library/cpp/scheme/scimpl_defs.h b/library/cpp/scheme/scimpl_defs.h index f3dd66b437..e6420194c8 100644 --- a/library/cpp/scheme/scimpl_defs.h +++ b/library/cpp/scheme/scimpl_defs.h @@ -20,17 +20,17 @@ namespace NSc { namespace NDefinitions { const size_t POOL_BLOCK_SIZE = 4000; // leave 96 bytes for overhead - struct TPool: public TAtomicRefCount<TPool> { + struct TPool: public TAtomicRefCount<TPool> { TMemoryPool Pool; TPool(size_t blsz = POOL_BLOCK_SIZE, TMemoryPool::IGrowPolicy* grow = TMemoryPool::TExpGrow::Instance()) : Pool(blsz, grow) - { - } + { + } - TMemoryPool* Get() { - return &Pool; - } + TMemoryPool* Get() { + return &Pool; + } TStringBuf AppendBuf(const TStringBuf& strb) { return Pool.AppendCString(strb); @@ -59,11 +59,11 @@ namespace NSc { TSchemeParseException(size_t off, const TString& reason) : Offset(off) , Reason(reason) - { - } + { + } }; - struct TJsonOpts: public NJson::TJsonReaderConfig { + struct TJsonOpts: public NJson::TJsonReaderConfig { enum EJsonOpts { JO_DEFAULT = 0, // just dump json, used to be default, actually JO_SORT_KEYS = 1, // sort dict keys to make output more predictable @@ -71,8 +71,8 @@ namespace NSc { // will skip nonunicode dict keys and replace nonunicode values with nulls JO_FORMAT = 8, // format json - JO_PARSER_STRICT_JSON = 16, // strict standard json - JO_PARSER_STRICT_UTF8 = 32, // strict utf8 + JO_PARSER_STRICT_JSON = 16, // strict standard json + JO_PARSER_STRICT_UTF8 = 32, // strict utf8 JO_PARSER_DISALLOW_COMMENTS = 64, JO_PARSER_DISALLOW_DUPLICATE_KEYS = 128, @@ -107,14 +107,14 @@ namespace NSc { public: static bool NumberPolicySafe(double&); // skip if nan or inf - static bool NumberPolicyUnsafe(double&) { - return true; - } + static bool NumberPolicyUnsafe(double&) { + return true; + } static bool StringPolicySafe(TStringBuf&); // skip if not utf8 - static bool StringPolicyUnsafe(TStringBuf&) { - return true; - } + static bool StringPolicyUnsafe(TStringBuf&) { + return true; + } }; struct TProtoOpts { diff --git a/library/cpp/scheme/scimpl_json_read.cpp b/library/cpp/scheme/scimpl_json_read.cpp index 8a29cc7739..8f289a8344 100644 --- a/library/cpp/scheme/scimpl_json_read.cpp +++ b/library/cpp/scheme/scimpl_json_read.cpp @@ -19,8 +19,8 @@ namespace NSc { TContainer(TValue& v) : Container(&v) , ExpectKey(v.IsDict()) - { - } + { + } bool Add(TStringBuf v, bool allowDuplicated) { if (!ExpectKey || Y_UNLIKELY(!Container->IsDict())) diff --git a/library/cpp/scheme/scimpl_json_write.cpp b/library/cpp/scheme/scimpl_json_write.cpp index aadd7e6cd5..79a1e3cef3 100644 --- a/library/cpp/scheme/scimpl_json_write.cpp +++ b/library/cpp/scheme/scimpl_json_write.cpp @@ -40,7 +40,7 @@ namespace NSc { template <typename TDictKeys> static inline void WriteDict(IOutputStream& out, const TDictKeys& keys, const TDict& dict, - const TJsonOpts& jopts, NImpl::TKeySortContext& sortCtx, NImpl::TSelfLoopContext& loopCtx) { + const TJsonOpts& jopts, NImpl::TKeySortContext& sortCtx, NImpl::TSelfLoopContext& loopCtx) { using const_iterator = typename TDictKeys::const_iterator; const_iterator begin = keys.begin(); const_iterator end = keys.end(); @@ -64,7 +64,7 @@ namespace NSc { } void TValue::DoWriteJsonImpl(IOutputStream& out, const TJsonOpts& jopts, - NImpl::TKeySortContext& sortCtx, NImpl::TSelfLoopContext& loopCtx) const { + NImpl::TKeySortContext& sortCtx, NImpl::TSelfLoopContext& loopCtx) const { const TScCore& core = Core(); NImpl::TSelfLoopContext::TGuard loopCheck(loopCtx, core); @@ -75,67 +75,67 @@ namespace NSc { } switch (core.ValueType) { - default: { - Y_ASSERT(false); + default: { + Y_ASSERT(false); [[fallthrough]]; /* no break */ } - case EType::Null: { + case EType::Null: { out << TStringBuf("null"); - break; + break; } - case EType::Bool: { + case EType::Bool: { out << (core.IntNumber ? TStringBuf("true") : TStringBuf("false")); - break; - } - case EType::IntNumber: { - out << core.IntNumber; - break; - } - case EType::FloatNumber: { - double d = core.FloatNumber; - if (!jopts.NumberPolicy || jopts.NumberPolicy(d)) { - out << d; - } else { + break; + } + case EType::IntNumber: { + out << core.IntNumber; + break; + } + case EType::FloatNumber: { + double d = core.FloatNumber; + if (!jopts.NumberPolicy || jopts.NumberPolicy(d)) { + out << d; + } else { out << TStringBuf("null"); } - break; - } - case EType::String: { - TStringBuf s = core.String; - if (!jopts.StringPolicy || jopts.StringPolicy(s)) { - WriteString(out, s); - } else { + break; + } + case EType::String: { + TStringBuf s = core.String; + if (!jopts.StringPolicy || jopts.StringPolicy(s)) { + WriteString(out, s); + } else { out << TStringBuf("null"); - } - break; - } - case EType::Array: { - out << '['; - const TArray& a = core.GetArray(); - for (TArray::const_iterator it = a.begin(); it != a.end(); ++it) { - if (it != a.begin()) { - out << ','; - } - - it->DoWriteJsonImpl(out, jopts, sortCtx, loopCtx); - } - out << ']'; - break; + } + break; + } + case EType::Array: { + out << '['; + const TArray& a = core.GetArray(); + for (TArray::const_iterator it = a.begin(); it != a.end(); ++it) { + if (it != a.begin()) { + out << ','; + } + + it->DoWriteJsonImpl(out, jopts, sortCtx, loopCtx); + } + out << ']'; + break; } - case EType::Dict: { - out << '{'; - - const TDict& dict = core.GetDict(); - - if (jopts.SortKeys) { - NImpl::TKeySortContext::TGuard keys(sortCtx, dict); - WriteDict(out, keys, dict, jopts, sortCtx, loopCtx); - } else { - WriteDict(out, dict, dict, jopts, sortCtx, loopCtx); - } - - out << '}'; - break; + case EType::Dict: { + out << '{'; + + const TDict& dict = core.GetDict(); + + if (jopts.SortKeys) { + NImpl::TKeySortContext::TGuard keys(sortCtx, dict); + WriteDict(out, keys, dict, jopts, sortCtx, loopCtx); + } else { + WriteDict(out, dict, dict, jopts, sortCtx, loopCtx); + } + + out << '}'; + break; } } } diff --git a/library/cpp/scheme/scimpl_private.cpp b/library/cpp/scheme/scimpl_private.cpp index 024bf8cc3b..417e27c49a 100644 --- a/library/cpp/scheme/scimpl_private.cpp +++ b/library/cpp/scheme/scimpl_private.cpp @@ -62,7 +62,7 @@ namespace NSc { break; case EMode::Stderr: Cerr << "REFERENCE LOOP DETECTED: " << JoinStrings(Vector.begin(), Vector.end(), ", ") - << " AND " << ToString((const void*)&self) << Endl; + << " AND " << ToString((const void*)&self) << Endl; break; } } diff --git a/library/cpp/scheme/scimpl_private.h b/library/cpp/scheme/scimpl_private.h index b92badabde..77f61e6bc4 100644 --- a/library/cpp/scheme/scimpl_private.h +++ b/library/cpp/scheme/scimpl_private.h @@ -16,7 +16,7 @@ namespace NSc { using TElement = typename TContext::TElement; using TTarget = typename TContext::TTarget; using TVectorType = TVector<TElement>; - + public: TContextGuard(TContext& ctx, TTarget& target) : Ctx(ctx) @@ -64,14 +64,14 @@ namespace NSc { using TElement = TElem; using TTarget = TTgt; - TBasicContext() { + TBasicContext() { Vector.reserve(64); } TVector<TElement> Vector; }; - class TKeySortContext: public TBasicContext<TStringBuf, const TDict> { + class TKeySortContext: public TBasicContext<TStringBuf, const TDict> { public: using TGuard = TContextGuard<TKeySortContext>; @@ -82,7 +82,7 @@ namespace NSc { } }; - class TSelfOverrideContext: public TBasicContext<TValue, TValue::TScCore> { + class TSelfOverrideContext: public TBasicContext<TValue, TValue::TScCore> { public: using TGuard = TContextGuard<TSelfOverrideContext>; @@ -93,7 +93,7 @@ namespace NSc { } }; - class TSelfLoopContext: public TBasicContext<const void*, const TValue::TScCore> { + class TSelfLoopContext: public TBasicContext<const void*, const TValue::TScCore> { public: enum class EMode { Assert, Throw, Abort, Stderr diff --git a/library/cpp/scheme/scimpl_protobuf.cpp b/library/cpp/scheme/scimpl_protobuf.cpp index 0c99122c69..32f73451e0 100644 --- a/library/cpp/scheme/scimpl_protobuf.cpp +++ b/library/cpp/scheme/scimpl_protobuf.cpp @@ -50,38 +50,38 @@ namespace NSc { const Reflection* r = msg.GetReflection(); switch (field->cpp_type()) { - case FieldDescriptor::CPPTYPE_INT32: - v = r->GetInt32(msg, field); - break; - case FieldDescriptor::CPPTYPE_INT64: - v = r->GetInt64(msg, field); - break; - case FieldDescriptor::CPPTYPE_UINT32: - v = r->GetUInt32(msg, field); - break; - case FieldDescriptor::CPPTYPE_UINT64: - v = r->GetUInt64(msg, field); - break; - case FieldDescriptor::CPPTYPE_DOUBLE: - v = r->GetDouble(msg, field); - break; - case FieldDescriptor::CPPTYPE_FLOAT: - v = r->GetFloat(msg, field); - break; - case FieldDescriptor::CPPTYPE_BOOL: - v.SetBool(r->GetBool(msg, field)); - break; - case FieldDescriptor::CPPTYPE_ENUM: - v = r->GetEnum(msg, field)->name(); - break; - case FieldDescriptor::CPPTYPE_STRING: - v = r->GetString(msg, field); - break; - case FieldDescriptor::CPPTYPE_MESSAGE: - v = From(r->GetMessage(msg, field)); - break; - default: - ythrow TSchemeException() << "field " << field->full_name() << " unexpected type " << (int)field->cpp_type(); + case FieldDescriptor::CPPTYPE_INT32: + v = r->GetInt32(msg, field); + break; + case FieldDescriptor::CPPTYPE_INT64: + v = r->GetInt64(msg, field); + break; + case FieldDescriptor::CPPTYPE_UINT32: + v = r->GetUInt32(msg, field); + break; + case FieldDescriptor::CPPTYPE_UINT64: + v = r->GetUInt64(msg, field); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + v = r->GetDouble(msg, field); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + v = r->GetFloat(msg, field); + break; + case FieldDescriptor::CPPTYPE_BOOL: + v.SetBool(r->GetBool(msg, field)); + break; + case FieldDescriptor::CPPTYPE_ENUM: + v = r->GetEnum(msg, field)->name(); + break; + case FieldDescriptor::CPPTYPE_STRING: + v = r->GetString(msg, field); + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + v = From(r->GetMessage(msg, field)); + break; + default: + ythrow TSchemeException() << "field " << field->full_name() << " unexpected type " << (int)field->cpp_type(); } return v; @@ -92,38 +92,38 @@ namespace NSc { const Reflection* r = msg.GetReflection(); switch (field->cpp_type()) { - case FieldDescriptor::CPPTYPE_INT32: - v = r->GetRepeatedInt32(msg, field, index); - break; - case FieldDescriptor::CPPTYPE_INT64: - v = r->GetRepeatedInt64(msg, field, index); - break; - case FieldDescriptor::CPPTYPE_UINT32: - v = r->GetRepeatedUInt32(msg, field, index); - break; - case FieldDescriptor::CPPTYPE_UINT64: - v = r->GetRepeatedUInt64(msg, field, index); - break; - case FieldDescriptor::CPPTYPE_DOUBLE: - v = r->GetRepeatedDouble(msg, field, index); - break; - case FieldDescriptor::CPPTYPE_FLOAT: - v = r->GetRepeatedFloat(msg, field, index); - break; - case FieldDescriptor::CPPTYPE_BOOL: - v.SetBool(r->GetRepeatedBool(msg, field, index)); - break; - case FieldDescriptor::CPPTYPE_ENUM: - v = r->GetRepeatedEnum(msg, field, index)->name(); - break; - case FieldDescriptor::CPPTYPE_STRING: - v = r->GetRepeatedString(msg, field, index); - break; - case FieldDescriptor::CPPTYPE_MESSAGE: - v = From(r->GetRepeatedMessage(msg, field, index)); - break; - default: - ythrow TSchemeException() << "field " << field->full_name() << " unexpected type " << (int)field->cpp_type(); + case FieldDescriptor::CPPTYPE_INT32: + v = r->GetRepeatedInt32(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_INT64: + v = r->GetRepeatedInt64(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_UINT32: + v = r->GetRepeatedUInt32(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_UINT64: + v = r->GetRepeatedUInt64(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + v = r->GetRepeatedDouble(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + v = r->GetRepeatedFloat(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_BOOL: + v.SetBool(r->GetRepeatedBool(msg, field, index)); + break; + case FieldDescriptor::CPPTYPE_ENUM: + v = r->GetRepeatedEnum(msg, field, index)->name(); + break; + case FieldDescriptor::CPPTYPE_STRING: + v = r->GetRepeatedString(msg, field, index); + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + v = From(r->GetRepeatedMessage(msg, field, index)); + break; + default: + ythrow TSchemeException() << "field " << field->full_name() << " unexpected type " << (int)field->cpp_type(); } return v; @@ -165,40 +165,40 @@ namespace NSc { const Reflection* reflection = msg.GetReflection(); switch (field->cpp_type()) { - case FieldDescriptor::CPPTYPE_INT32: - reflection->SetInt32(&msg, field, value.ForceIntNumber()); - break; - case FieldDescriptor::CPPTYPE_INT64: - reflection->SetInt64(&msg, field, value.ForceIntNumber()); - break; - case FieldDescriptor::CPPTYPE_UINT32: - reflection->SetUInt32(&msg, field, value.ForceIntNumber()); - break; - case FieldDescriptor::CPPTYPE_UINT64: - reflection->SetUInt64(&msg, field, value.ForceIntNumber()); - break; - case FieldDescriptor::CPPTYPE_DOUBLE: - reflection->SetDouble(&msg, field, value.ForceNumber()); - break; - case FieldDescriptor::CPPTYPE_FLOAT: - reflection->SetFloat(&msg, field, value.ForceNumber()); - break; - case FieldDescriptor::CPPTYPE_BOOL: - reflection->SetBool(&msg, field, value.IsTrue()); - break; - case FieldDescriptor::CPPTYPE_STRING: - reflection->SetString(&msg, field, value.ForceString()); - break; - case FieldDescriptor::CPPTYPE_ENUM: + case FieldDescriptor::CPPTYPE_INT32: + reflection->SetInt32(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_INT64: + reflection->SetInt64(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_UINT32: + reflection->SetUInt32(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_UINT64: + reflection->SetUInt64(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + reflection->SetDouble(&msg, field, value.ForceNumber()); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + reflection->SetFloat(&msg, field, value.ForceNumber()); + break; + case FieldDescriptor::CPPTYPE_BOOL: + reflection->SetBool(&msg, field, value.IsTrue()); + break; + case FieldDescriptor::CPPTYPE_STRING: + reflection->SetString(&msg, field, value.ForceString()); + break; + case FieldDescriptor::CPPTYPE_ENUM: value.ToEnumField(msg, field, opts); - break; - case FieldDescriptor::CPPTYPE_MESSAGE: + break; + case FieldDescriptor::CPPTYPE_MESSAGE: value.To(*reflection->MutableMessage(&msg, field), opts); - break; - default: - ythrow TSchemeException() - << "field " << field->full_name() - << " unexpected type " << (int)field->cpp_type(); + break; + default: + ythrow TSchemeException() + << "field " << field->full_name() + << " unexpected type " << (int)field->cpp_type(); } } @@ -212,8 +212,8 @@ namespace NSc { const EnumDescriptor* enumField = field->enum_type(); const EnumValueDescriptor* enumFieldValue = IsString() - ? enumField->FindValueByName(ForceString()) - : enumField->FindValueByNumber(ForceIntNumber()); + ? enumField->FindValueByName(ForceString()) + : enumField->FindValueByNumber(ForceIntNumber()); if (!enumFieldValue) { if (opts.UnknownEnumValueIsDefault) { @@ -250,42 +250,42 @@ namespace NSc { const Reflection* reflection = msg.GetReflection(); - for (const TValue& value : fieldValue.GetArray()) { + for (const TValue& value : fieldValue.GetArray()) { switch (field->cpp_type()) { - case FieldDescriptor::CPPTYPE_INT32: - reflection->AddInt32(&msg, field, value.ForceIntNumber()); - break; - case FieldDescriptor::CPPTYPE_INT64: - reflection->AddInt64(&msg, field, value.ForceIntNumber()); - break; - case FieldDescriptor::CPPTYPE_UINT32: - reflection->AddUInt32(&msg, field, value.ForceIntNumber()); - break; - case FieldDescriptor::CPPTYPE_UINT64: - reflection->AddUInt64(&msg, field, value.ForceIntNumber()); - break; - case FieldDescriptor::CPPTYPE_DOUBLE: - reflection->AddDouble(&msg, field, value.ForceNumber()); - break; - case FieldDescriptor::CPPTYPE_FLOAT: - reflection->AddFloat(&msg, field, value.ForceNumber()); - break; - case FieldDescriptor::CPPTYPE_BOOL: - reflection->AddBool(&msg, field, value.IsTrue()); - break; - case FieldDescriptor::CPPTYPE_STRING: - reflection->AddString(&msg, field, value.ForceString()); - break; - case FieldDescriptor::CPPTYPE_ENUM: + case FieldDescriptor::CPPTYPE_INT32: + reflection->AddInt32(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_INT64: + reflection->AddInt64(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_UINT32: + reflection->AddUInt32(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_UINT64: + reflection->AddUInt64(&msg, field, value.ForceIntNumber()); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + reflection->AddDouble(&msg, field, value.ForceNumber()); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + reflection->AddFloat(&msg, field, value.ForceNumber()); + break; + case FieldDescriptor::CPPTYPE_BOOL: + reflection->AddBool(&msg, field, value.IsTrue()); + break; + case FieldDescriptor::CPPTYPE_STRING: + reflection->AddString(&msg, field, value.ForceString()); + break; + case FieldDescriptor::CPPTYPE_ENUM: value.ToEnumField(msg, field, opts); - break; - case FieldDescriptor::CPPTYPE_MESSAGE: - value.To(*reflection->AddMessage(&msg, field)); - break; - default: - ythrow TSchemeException() - << "field " << field->full_name() - << " unexpected type " << (int)field->cpp_type(); + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + value.To(*reflection->AddMessage(&msg, field)); + break; + default: + ythrow TSchemeException() + << "field " << field->full_name() + << " unexpected type " << (int)field->cpp_type(); } } } diff --git a/library/cpp/scheme/tests/ut/scheme_cast_ut.cpp b/library/cpp/scheme/tests/ut/scheme_cast_ut.cpp index 4f907157e9..a57b77e566 100644 --- a/library/cpp/scheme/tests/ut/scheme_cast_ut.cpp +++ b/library/cpp/scheme/tests/ut/scheme_cast_ut.cpp @@ -26,13 +26,13 @@ Y_UNIT_TEST_SUITE(TSchemeCastTest) { TVI y(FromJson<TVI>(ToJson(v))); UNIT_ASSERT_VALUES_EQUAL(v.size(), y.size()); - UNIT_ASSERT(std::equal(v.begin(), v.end(), y.begin())); + UNIT_ASSERT(std::equal(v.begin(), v.end(), y.begin())); } Y_UNIT_TEST(TestYHash) { THI h; for (int i = 0; i < 3; ++i) - h[i] = i * i; + h[i] = i * i; const TString etalon = "{\"0\":0,\"1\":1,\"2\":4}"; UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(h, true)); @@ -44,7 +44,7 @@ Y_UNIT_TEST_SUITE(TSchemeCastTest) { Y_UNIT_TEST(TestYMap) { TMI h; for (int i = 0; i < 3; ++i) - h[i] = i * i; + h[i] = i * i; const TString etalon = "{\"0\":0,\"1\":1,\"2\":4}"; UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(h, true)); @@ -56,7 +56,7 @@ Y_UNIT_TEST_SUITE(TSchemeCastTest) { Y_UNIT_TEST(TestYHashSet) { THSI h; for (int i = 0; i < 3; ++i) - h.insert(i * i); + h.insert(i * i); const TString etalon = "{\"0\":null,\"1\":null,\"4\":null}"; UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(h, true)); @@ -68,7 +68,7 @@ Y_UNIT_TEST_SUITE(TSchemeCastTest) { Y_UNIT_TEST(TestYSet) { TSI h; for (int i = 0; i < 3; ++i) - h.insert(i * i); + h.insert(i * i); const TString etalon = "{\"0\":null,\"1\":null,\"4\":null}"; UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(h, true)); @@ -78,7 +78,7 @@ Y_UNIT_TEST_SUITE(TSchemeCastTest) { } Y_UNIT_TEST(TestTPair) { - TPI p(1, 1); + TPI p(1, 1); const TString etalon = "[1,1]"; UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(p, true)); @@ -89,14 +89,14 @@ Y_UNIT_TEST_SUITE(TSchemeCastTest) { struct TCustom: public IJsonSerializable { int A, B; - TCustom() - : A(0) - , B(0){}; - TCustom(int a, int b) - : A(a) - , B(b) - { - } + TCustom() + : A(0) + , B(0){}; + TCustom(int a, int b) + : A(a) + , B(b) + { + } NSc::TValue ToTValue() const override { NSc::TValue res; res["a"] = A; @@ -108,7 +108,7 @@ Y_UNIT_TEST_SUITE(TSchemeCastTest) { B = v["b"].GetNumber(); } - bool operator<(const TCustom& rhs) const { + bool operator<(const TCustom& rhs) const { return A < rhs.A; } }; @@ -128,7 +128,7 @@ Y_UNIT_TEST_SUITE(TSchemeCastTest) { TVPI v; for (int i = 0; i < 3; ++i) - v.push_back(TPI(i, i * i)); + v.push_back(TPI(i, i * i)); const TString etalon = "[[0,0],[1,1],[2,4]]"; UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(v, true)); @@ -140,7 +140,7 @@ Y_UNIT_TEST_SUITE(TSchemeCastTest) { Y_UNIT_TEST(TestSetOfCustom) { typedef TSet<TCustom> TSC; TSC s; - s.insert(TCustom(2, 3)); + s.insert(TCustom(2, 3)); const TString etalon = "{\"{\\\"a\\\":2,\\\"b\\\":3}\":null}"; UNIT_ASSERT_VALUES_EQUAL(etalon, ToJson(s, true)); diff --git a/library/cpp/scheme/tests/ut/scheme_json_ut.cpp b/library/cpp/scheme/tests/ut/scheme_json_ut.cpp index daeb2654f9..561829ffab 100644 --- a/library/cpp/scheme/tests/ut/scheme_json_ut.cpp +++ b/library/cpp/scheme/tests/ut/scheme_json_ut.cpp @@ -14,12 +14,12 @@ using namespace std::string_view_literals; Y_UNIT_TEST_SUITE(TSchemeJsonTest) { Y_UNIT_TEST(TestJson) { const char* json = "[\n" - " {\n" - " \"url\":\"foo\",\n" - " \"title\":\"bar\",\n" - " \"passages\":[\"foo\", \"bar\"],\n" - " }\n" - "]"; + " {\n" + " \"url\":\"foo\",\n" + " \"title\":\"bar\",\n" + " \"passages\":[\"foo\", \"bar\"],\n" + " }\n" + "]"; { const NSc::TValue& v = NSc::TValue::FromJson(json); @@ -84,32 +84,32 @@ Y_UNIT_TEST_SUITE(TSchemeJsonTest) { UNIT_ASSERT_VALUES_EQUAL("{\"xxx\":null}", v.ToJson(NSc::TValue::JO_SAFE)); UNIT_ASSERT_VALUES_EQUAL("{" - "\"\\u0000\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\b\\t\\n\\u000B\\f\\r" - "\\u000E\\u000F\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018" - "\\u0019\\u001A\\u001B\\u001C\\u001D\\u001E\\u001F !\\\"#$%&'()*+,-./0123456789" - ":;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\u007F" - "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93" - "\x94\x95\x96\x97\x98\x99\x9A\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7" - "\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB" - "\xBC\xBD\xBE\xBF\\xC0\\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA\xCB\xCC\xCD\xCE" - "\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1" - "\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4" - "\\xF5\\xF6\\xF7\\xF8\\xF9\\xFA\\xFB\\xFC\\xFD\\xFE\\xFF\":" - "\"xxx\"," - "\"xxx\":" - "\"\\u0000\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\b\\t\\n\\u000B\\f\\r" - "\\u000E\\u000F\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018" - "\\u0019\\u001A\\u001B\\u001C\\u001D\\u001E\\u001F !\\\"#$%&'()*+,-./0123456789" - ":;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\u007F" - "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93" - "\x94\x95\x96\x97\x98\x99\x9A\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7" - "\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB" - "\xBC\xBD\xBE\xBF\\xC0\\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA\xCB\xCC\xCD\xCE" - "\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1" - "\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4" - "\\xF5\\xF6\\xF7\\xF8\\xF9\\xFA\\xFB\\xFC\\xFD\\xFE\\xFF\"" - "}", - v.ToJson(NSc::TValue::JO_SORT_KEYS)); + "\"\\u0000\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\b\\t\\n\\u000B\\f\\r" + "\\u000E\\u000F\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018" + "\\u0019\\u001A\\u001B\\u001C\\u001D\\u001E\\u001F !\\\"#$%&'()*+,-./0123456789" + ":;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\u007F" + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93" + "\x94\x95\x96\x97\x98\x99\x9A\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7" + "\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB" + "\xBC\xBD\xBE\xBF\\xC0\\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA\xCB\xCC\xCD\xCE" + "\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1" + "\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4" + "\\xF5\\xF6\\xF7\\xF8\\xF9\\xFA\\xFB\\xFC\\xFD\\xFE\\xFF\":" + "\"xxx\"," + "\"xxx\":" + "\"\\u0000\\u0001\\u0002\\u0003\\u0004\\u0005\\u0006\\u0007\\b\\t\\n\\u000B\\f\\r" + "\\u000E\\u000F\\u0010\\u0011\\u0012\\u0013\\u0014\\u0015\\u0016\\u0017\\u0018" + "\\u0019\\u001A\\u001B\\u001C\\u001D\\u001E\\u001F !\\\"#$%&'()*+,-./0123456789" + ":;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\\u007F" + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F\x90\x91\x92\x93" + "\x94\x95\x96\x97\x98\x99\x9A\x9B\x9C\x9D\x9E\x9F\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7" + "\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB" + "\xBC\xBD\xBE\xBF\\xC0\\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA\xCB\xCC\xCD\xCE" + "\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1" + "\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4" + "\\xF5\\xF6\\xF7\\xF8\\xF9\\xFA\\xFB\\xFC\\xFD\\xFE\\xFF\"" + "}", + v.ToJson(NSc::TValue::JO_SORT_KEYS)); UNIT_ASSERT(NSc::TValue::Equal(v, NSc::TValue::FromJson(v.ToJson()))); { @@ -152,7 +152,7 @@ Y_UNIT_TEST_SUITE(TSchemeJsonTest) { } Y_UNIT_TEST(TestDuplicateKeys) { - const TStringBuf duplicatedKeys = "{\"a\":[{\"b\":1, \"b\":42}]}"; + const TStringBuf duplicatedKeys = "{\"a\":[{\"b\":1, \"b\":42}]}"; UNIT_ASSERT_NO_EXCEPTION(NSc::TValue::FromJsonThrow(duplicatedKeys)); UNIT_ASSERT_EXCEPTION(NSc::TValue::FromJsonThrow(duplicatedKeys, NSc::TValue::JO_PARSER_DISALLOW_DUPLICATE_KEYS), yexception); UNIT_ASSERT(NSc::TValue::FromJson(duplicatedKeys).IsDict()); diff --git a/library/cpp/scheme/tests/ut/scheme_path_ut.cpp b/library/cpp/scheme/tests/ut/scheme_path_ut.cpp index 0d4d79d483..45cd19a4be 100644 --- a/library/cpp/scheme/tests/ut/scheme_path_ut.cpp +++ b/library/cpp/scheme/tests/ut/scheme_path_ut.cpp @@ -34,21 +34,21 @@ Y_UNIT_TEST_SUITE(TSchemePathTest) { UNIT_ASSERT(v.PathExists("//")); UNIT_ASSERT(NSc::TValue::Same(v, *v.TrySelectOrAdd("//"))); - NSc::NUt::AssertSchemeJson("null", v); + NSc::NUt::AssertSchemeJson("null", v); UNIT_ASSERT(NSc::TValue::Same(v.TrySelectAndDelete("//"), NSc::Null())); - NSc::NUt::AssertSchemeJson("null", v); + NSc::NUt::AssertSchemeJson("null", v); v.SetDict(); UNIT_ASSERT(NSc::TValue::Same(v, *v.TrySelectOrAdd("//"))); - NSc::NUt::AssertSchemeJson("{}", v); + NSc::NUt::AssertSchemeJson("{}", v); UNIT_ASSERT(NSc::TValue::Same(v.TrySelectAndDelete("//"), NSc::Null())); - NSc::NUt::AssertSchemeJson("{}", v); + NSc::NUt::AssertSchemeJson("{}", v); v.SetArray(); UNIT_ASSERT(NSc::TValue::Same(v, *v.TrySelectOrAdd("//"))); - NSc::NUt::AssertSchemeJson("[]", v); + NSc::NUt::AssertSchemeJson("[]", v); UNIT_ASSERT(NSc::TValue::Same(v.TrySelectAndDelete("//"), NSc::Null())); - NSc::NUt::AssertSchemeJson("[]", v); + NSc::NUt::AssertSchemeJson("[]", v); DoTestSelect("[]", "{'':1}", "{}"); DoTestSelect("[ ]", "{' ':1}", "{}"); @@ -69,27 +69,27 @@ Y_UNIT_TEST_SUITE(TSchemePathTest) { } Y_UNIT_TEST(TestPathEscapes) { - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a"), "a"); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath(""), R"=("")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[]"), R"=("[]")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a"), "a"); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath(""), R"=("")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[]"), R"=("[]")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[]ab"), R"=("[]ab")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a[]b"), R"=("a[]b")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab[]"), R"=("ab[]")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[ab]"), R"=("[ab]")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[ab"), R"=("[ab")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a[b"), R"=("a[b")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab["), R"=("ab[")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("]ab"), R"=("]ab")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a]b"), R"=("a]b")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab]"), R"=("ab]")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath(R"=(\)="), R"=("\\")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("[ab"), R"=("[ab")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a[b"), R"=("a[b")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab["), R"=("ab[")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("]ab"), R"=("]ab")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a]b"), R"=("a]b")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab]"), R"=("ab]")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath(R"=(\)="), R"=("\\")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath(R"=(\\)="), R"=("\\\\")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("/"), R"=("/")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("//"), R"=("//")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("///"), R"=("///")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("/ab"), R"=("/ab")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a/b"), R"=("a/b")="); - UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab/"), R"=("ab/")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("/"), R"=("/")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("//"), R"=("//")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("///"), R"=("///")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("/ab"), R"=("/ab")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a/b"), R"=("a/b")="); + UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab/"), R"=("ab/")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("//ab"), R"=("//ab")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("a//b"), R"=("a//b")="); UNIT_ASSERT_VALUES_EQUAL(NSc::TValue::EscapeForPath("ab//"), R"=("ab//")="); diff --git a/library/cpp/scheme/tests/ut/scheme_proto_ut.cpp b/library/cpp/scheme/tests/ut/scheme_proto_ut.cpp index e711a0d092..502c74b15b 100644 --- a/library/cpp/scheme/tests/ut/scheme_proto_ut.cpp +++ b/library/cpp/scheme/tests/ut/scheme_proto_ut.cpp @@ -36,22 +36,22 @@ Y_UNIT_TEST_SUITE(TSchemeProtoTest) { void DoTestProtobuf(bool fromProto, bool mapAsDict) { NSc::TMessage m; NSc::TValue v; - m.SetDouble((double)1 / 3), v["Double"] = (double)1 / 3; - m.SetFloat((float)1 / 3), v["Float"] = (float)1 / 3; - m.SetInt32(1000000), v["Int32"] = 1000000; - m.SetInt64(1000000000000LL), v["Int64"] = 1000000000000LL; - m.SetUInt32(555555), v["UInt32"] = 555555; - m.SetUInt64(555555555555LL), v["UInt64"] = 555555555555LL; - m.SetSInt32(-555555), v["SInt32"] = -555555; - m.SetSInt64(-555555555555LL), v["SInt64"] = -555555555555LL; - m.SetFixed32(123456), v["Fixed32"] = 123456; - m.SetFixed64(123456123456LL), v["Fixed64"] = 123456123456LL; - m.SetSFixed32(-123456), v["SFixed32"] = -123456; - m.SetSFixed64(-123456123456LL), v["SFixed64"] = -123456123456LL; - m.SetBool(true), v["Bool"] = true; - m.SetString("String"), v["String"] = "String"; - m.SetBytes("Bytes"), v["Bytes"] = "Bytes"; - m.SetEnum(NSc::VALUE1), v["Enum"] = "VALUE1"; + m.SetDouble((double)1 / 3), v["Double"] = (double)1 / 3; + m.SetFloat((float)1 / 3), v["Float"] = (float)1 / 3; + m.SetInt32(1000000), v["Int32"] = 1000000; + m.SetInt64(1000000000000LL), v["Int64"] = 1000000000000LL; + m.SetUInt32(555555), v["UInt32"] = 555555; + m.SetUInt64(555555555555LL), v["UInt64"] = 555555555555LL; + m.SetSInt32(-555555), v["SInt32"] = -555555; + m.SetSInt64(-555555555555LL), v["SInt64"] = -555555555555LL; + m.SetFixed32(123456), v["Fixed32"] = 123456; + m.SetFixed64(123456123456LL), v["Fixed64"] = 123456123456LL; + m.SetSFixed32(-123456), v["SFixed32"] = -123456; + m.SetSFixed64(-123456123456LL), v["SFixed64"] = -123456123456LL; + m.SetBool(true), v["Bool"] = true; + m.SetString("String"), v["String"] = "String"; + m.SetBytes("Bytes"), v["Bytes"] = "Bytes"; + m.SetEnum(NSc::VALUE1), v["Enum"] = "VALUE1"; auto& mapDoublesP = *m.mutable_mapdoubles(); auto& mapDoublesV = v["MapDoubles"]; @@ -83,129 +83,129 @@ Y_UNIT_TEST_SUITE(TSchemeProtoTest) { AddMapPair(mapStringV, "deep", "blue", mapAsDict); } - m.AddDoubles((double)1 / 3), v["Doubles"][0] = (double)1 / 3; - m.AddDoubles((double)1 / 4), v["Doubles"][1] = (double)1 / 4; + m.AddDoubles((double)1 / 3), v["Doubles"][0] = (double)1 / 3; + m.AddDoubles((double)1 / 4), v["Doubles"][1] = (double)1 / 4; - m.AddFloats((float)1 / 3), v["Floats"][0] = (float)1 / 3; - m.AddFloats((float)1 / 4), v["Floats"][1] = (float)1 / 4; + m.AddFloats((float)1 / 3), v["Floats"][0] = (float)1 / 3; + m.AddFloats((float)1 / 4), v["Floats"][1] = (float)1 / 4; - m.AddInt32s(1000000), v["Int32s"][0] = 1000000; - m.AddInt32s(2000000), v["Int32s"][1] = 2000000; + m.AddInt32s(1000000), v["Int32s"][0] = 1000000; + m.AddInt32s(2000000), v["Int32s"][1] = 2000000; - m.AddInt64s(1000000000000LL), v["Int64s"][0] = 1000000000000LL; - m.AddInt64s(2000000000000LL), v["Int64s"][1] = 2000000000000LL; + m.AddInt64s(1000000000000LL), v["Int64s"][0] = 1000000000000LL; + m.AddInt64s(2000000000000LL), v["Int64s"][1] = 2000000000000LL; - m.AddUInt32s(555555), v["UInt32s"][0] = 555555; - m.AddUInt32s(655555), v["UInt32s"][1] = 655555; + m.AddUInt32s(555555), v["UInt32s"][0] = 555555; + m.AddUInt32s(655555), v["UInt32s"][1] = 655555; - m.AddUInt64s(555555555555LL); - v["UInt64s"][0] = 555555555555LL; - m.AddUInt64s(655555555555LL); - v["UInt64s"][1] = 655555555555LL; + m.AddUInt64s(555555555555LL); + v["UInt64s"][0] = 555555555555LL; + m.AddUInt64s(655555555555LL); + v["UInt64s"][1] = 655555555555LL; - m.AddSInt32s(-555555), v["SInt32s"][0] = -555555; - m.AddSInt32s(-655555), v["SInt32s"][1] = -655555; + m.AddSInt32s(-555555), v["SInt32s"][0] = -555555; + m.AddSInt32s(-655555), v["SInt32s"][1] = -655555; - m.AddSInt64s(-555555555555LL), v["SInt64s"][0] = -555555555555LL; - m.AddSInt64s(-655555555555LL), v["SInt64s"][1] = -655555555555LL; + m.AddSInt64s(-555555555555LL), v["SInt64s"][0] = -555555555555LL; + m.AddSInt64s(-655555555555LL), v["SInt64s"][1] = -655555555555LL; - m.AddFixed32s(123456), v["Fixed32s"][0] = 123456; - m.AddFixed32s(223456), v["Fixed32s"][1] = 223456; + m.AddFixed32s(123456), v["Fixed32s"][0] = 123456; + m.AddFixed32s(223456), v["Fixed32s"][1] = 223456; - m.AddFixed64s(123456123456LL), v["Fixed64s"][0] = 123456123456LL; - m.AddFixed64s(223456123456LL), v["Fixed64s"][1] = 223456123456LL; + m.AddFixed64s(123456123456LL), v["Fixed64s"][0] = 123456123456LL; + m.AddFixed64s(223456123456LL), v["Fixed64s"][1] = 223456123456LL; - m.AddSFixed32s(-123456), v["SFixed32s"][0] = -123456; - m.AddSFixed32s(-223456), v["SFixed32s"][1] = -223456; + m.AddSFixed32s(-123456), v["SFixed32s"][0] = -123456; + m.AddSFixed32s(-223456), v["SFixed32s"][1] = -223456; - m.AddSFixed64s(-123456123456LL), v["SFixed64s"][0] = -123456123456LL; - m.AddSFixed64s(-223456123456LL), v["SFixed64s"][1] = -223456123456LL; + m.AddSFixed64s(-123456123456LL), v["SFixed64s"][0] = -123456123456LL; + m.AddSFixed64s(-223456123456LL), v["SFixed64s"][1] = -223456123456LL; - m.AddBools(false), v["Bools"][0] = false; - m.AddBools(true), v["Bools"][1] = true; + m.AddBools(false), v["Bools"][0] = false; + m.AddBools(true), v["Bools"][1] = true; - m.AddStrings("String1"), v["Strings"][0] = "String1"; - m.AddStrings("String2"), v["Strings"][1] = "String2"; + m.AddStrings("String1"), v["Strings"][0] = "String1"; + m.AddStrings("String2"), v["Strings"][1] = "String2"; - m.AddBytess("Bytes1"), v["Bytess"][0] = "Bytes1"; - m.AddBytess("Bytes2"), v["Bytess"][1] = "Bytes2"; + m.AddBytess("Bytes1"), v["Bytess"][0] = "Bytes1"; + m.AddBytess("Bytes2"), v["Bytess"][1] = "Bytes2"; - m.AddEnums(NSc::VALUE1), v["Enums"][0] = "VALUE1"; - m.AddEnums(NSc::VALUE2), v["Enums"][1] = "VALUE2"; + m.AddEnums(NSc::VALUE1), v["Enums"][0] = "VALUE1"; + m.AddEnums(NSc::VALUE2), v["Enums"][1] = "VALUE2"; NSc::TMessage2 m2; NSc::TValue v2; - m2.SetDouble((double)1 / 3), v2["Double"] = (double)1 / 3; - m2.SetFloat((float)1 / 3), v2["Float"] = (float)1 / 3; - m2.SetInt32(1000000), v2["Int32"] = 1000000; - m2.SetInt64(1000000000000LL), v2["Int64"] = 1000000000000LL; - m2.SetUInt32(555555), v2["UInt32"] = 555555; - m2.SetUInt64(555555555555LL), v2["UInt64"] = 555555555555LL; - m2.SetSInt32(-555555), v2["SInt32"] = -555555; - m2.SetSInt64(-555555555555LL), v2["SInt64"] = -555555555555LL; - m2.SetFixed32(123456), v2["Fixed32"] = 123456; - m2.SetFixed64(123456123456LL), v2["Fixed64"] = 123456123456LL; - m2.SetSFixed32(-123456), v2["SFixed32"] = -123456; - m2.SetSFixed64(-123456123456LL), v2["SFixed64"] = -123456123456LL; - m2.SetBool(true), v2["Bool"] = true; - m2.SetString("String"), v2["String"] = "String"; - m2.SetBytes("Bytes"), v2["Bytes"] = "Bytes"; - m2.SetEnum(NSc::VALUE1), v2["Enum"] = "VALUE1"; + m2.SetDouble((double)1 / 3), v2["Double"] = (double)1 / 3; + m2.SetFloat((float)1 / 3), v2["Float"] = (float)1 / 3; + m2.SetInt32(1000000), v2["Int32"] = 1000000; + m2.SetInt64(1000000000000LL), v2["Int64"] = 1000000000000LL; + m2.SetUInt32(555555), v2["UInt32"] = 555555; + m2.SetUInt64(555555555555LL), v2["UInt64"] = 555555555555LL; + m2.SetSInt32(-555555), v2["SInt32"] = -555555; + m2.SetSInt64(-555555555555LL), v2["SInt64"] = -555555555555LL; + m2.SetFixed32(123456), v2["Fixed32"] = 123456; + m2.SetFixed64(123456123456LL), v2["Fixed64"] = 123456123456LL; + m2.SetSFixed32(-123456), v2["SFixed32"] = -123456; + m2.SetSFixed64(-123456123456LL), v2["SFixed64"] = -123456123456LL; + m2.SetBool(true), v2["Bool"] = true; + m2.SetString("String"), v2["String"] = "String"; + m2.SetBytes("Bytes"), v2["Bytes"] = "Bytes"; + m2.SetEnum(NSc::VALUE1), v2["Enum"] = "VALUE1"; - m2.AddDoubles((double)1 / 3), v2["Doubles"][0] = (double)1 / 3; - m2.AddDoubles((double)1 / 4), v2["Doubles"][1] = (double)1 / 4; + m2.AddDoubles((double)1 / 3), v2["Doubles"][0] = (double)1 / 3; + m2.AddDoubles((double)1 / 4), v2["Doubles"][1] = (double)1 / 4; - m2.AddFloats((float)1 / 3), v2["Floats"][0] = (float)1 / 3; - m2.AddFloats((float)1 / 4), v2["Floats"][1] = (float)1 / 4; + m2.AddFloats((float)1 / 3), v2["Floats"][0] = (float)1 / 3; + m2.AddFloats((float)1 / 4), v2["Floats"][1] = (float)1 / 4; - m2.AddInt32s(1000000), v2["Int32s"][0] = 1000000; - m2.AddInt32s(2000000), v2["Int32s"][1] = 2000000; + m2.AddInt32s(1000000), v2["Int32s"][0] = 1000000; + m2.AddInt32s(2000000), v2["Int32s"][1] = 2000000; - m2.AddInt64s(1000000000000LL), v2["Int64s"][0] = 1000000000000LL; - m2.AddInt64s(2000000000000LL), v2["Int64s"][1] = 2000000000000LL; + m2.AddInt64s(1000000000000LL), v2["Int64s"][0] = 1000000000000LL; + m2.AddInt64s(2000000000000LL), v2["Int64s"][1] = 2000000000000LL; - m2.AddUInt32s(555555), v2["UInt32s"][0] = 555555; - m2.AddUInt32s(655555), v2["UInt32s"][1] = 655555; + m2.AddUInt32s(555555), v2["UInt32s"][0] = 555555; + m2.AddUInt32s(655555), v2["UInt32s"][1] = 655555; - m2.AddUInt64s(555555555555LL); - v2["UInt64s"][0] = 555555555555LL; - m2.AddUInt64s(655555555555LL); - v2["UInt64s"][1] = 655555555555LL; + m2.AddUInt64s(555555555555LL); + v2["UInt64s"][0] = 555555555555LL; + m2.AddUInt64s(655555555555LL); + v2["UInt64s"][1] = 655555555555LL; - m2.AddSInt32s(-555555), v2["SInt32s"][0] = -555555; - m2.AddSInt32s(-655555), v2["SInt32s"][1] = -655555; + m2.AddSInt32s(-555555), v2["SInt32s"][0] = -555555; + m2.AddSInt32s(-655555), v2["SInt32s"][1] = -655555; - m2.AddSInt64s(-555555555555LL), v2["SInt64s"][0] = -555555555555LL; - m2.AddSInt64s(-655555555555LL), v2["SInt64s"][1] = -655555555555LL; + m2.AddSInt64s(-555555555555LL), v2["SInt64s"][0] = -555555555555LL; + m2.AddSInt64s(-655555555555LL), v2["SInt64s"][1] = -655555555555LL; - m2.AddFixed32s(123456), v2["Fixed32s"][0] = 123456; - m2.AddFixed32s(223456), v2["Fixed32s"][1] = 223456; + m2.AddFixed32s(123456), v2["Fixed32s"][0] = 123456; + m2.AddFixed32s(223456), v2["Fixed32s"][1] = 223456; - m2.AddFixed64s(123456123456LL), v2["Fixed64s"][0] = 123456123456LL; - m2.AddFixed64s(223456123456LL), v2["Fixed64s"][1] = 223456123456LL; + m2.AddFixed64s(123456123456LL), v2["Fixed64s"][0] = 123456123456LL; + m2.AddFixed64s(223456123456LL), v2["Fixed64s"][1] = 223456123456LL; - m2.AddSFixed32s(-123456), v2["SFixed32s"][0] = -123456; - m2.AddSFixed32s(-223456), v2["SFixed32s"][1] = -223456; + m2.AddSFixed32s(-123456), v2["SFixed32s"][0] = -123456; + m2.AddSFixed32s(-223456), v2["SFixed32s"][1] = -223456; - m2.AddSFixed64s(-123456123456LL), v2["SFixed64s"][0] = -123456123456LL; - m2.AddSFixed64s(-223456123456LL), v2["SFixed64s"][1] = -223456123456LL; + m2.AddSFixed64s(-123456123456LL), v2["SFixed64s"][0] = -123456123456LL; + m2.AddSFixed64s(-223456123456LL), v2["SFixed64s"][1] = -223456123456LL; - m2.AddBools(false), v2["Bools"][0] = false; - m2.AddBools(true), v2["Bools"][1] = true; + m2.AddBools(false), v2["Bools"][0] = false; + m2.AddBools(true), v2["Bools"][1] = true; - m2.AddStrings("String1"), v2["Strings"][0] = "String1"; - m2.AddStrings("String2"), v2["Strings"][1] = "String2"; + m2.AddStrings("String1"), v2["Strings"][0] = "String1"; + m2.AddStrings("String2"), v2["Strings"][1] = "String2"; - m2.AddBytess("Bytes1"), v2["Bytess"][0] = "Bytes1"; - m2.AddBytess("Bytes2"), v2["Bytess"][1] = "Bytes2"; + m2.AddBytess("Bytes1"), v2["Bytess"][0] = "Bytes1"; + m2.AddBytess("Bytes2"), v2["Bytess"][1] = "Bytes2"; - m2.AddEnums(NSc::VALUE1), v2["Enums"][0] = "VALUE1"; - m2.AddEnums(NSc::VALUE2), v2["Enums"][1] = "VALUE2"; + m2.AddEnums(NSc::VALUE1), v2["Enums"][0] = "VALUE1"; + m2.AddEnums(NSc::VALUE2), v2["Enums"][1] = "VALUE2"; - *(m.MutableMessage()) = m2, v["Message"] = v2; + *(m.MutableMessage()) = m2, v["Message"] = v2; - *(m.AddMessages()) = m2, v["Messages"][0] = v2; - *(m.AddMessages()) = m2, v["Messages"][1] = v2; + *(m.AddMessages()) = m2, v["Messages"][0] = v2; + *(m.AddMessages()) = m2, v["Messages"][1] = v2; if (fromProto) { UNIT_ASSERT(NSc::TValue::Equal(v, NSc::TValue::From(m, mapAsDict))); diff --git a/library/cpp/scheme/tests/ut/scheme_ut.cpp b/library/cpp/scheme/tests/ut/scheme_ut.cpp index 1a5d07c31b..a6bfc7d843 100644 --- a/library/cpp/scheme/tests/ut/scheme_ut.cpp +++ b/library/cpp/scheme/tests/ut/scheme_ut.cpp @@ -134,11 +134,11 @@ Y_UNIT_TEST_SUITE(TSchemeTest) { UNIT_ASSERT_VALUES_EQUAL_C(i64(t), NSc::TValue(i64(t)).ForceIntNumber(), ToString(t)); UNIT_ASSERT_VALUES_EQUAL_C(double(t), NSc::TValue(double(t)).ForceNumber(), ToString(t)); - UNIT_ASSERT_VALUES_EQUAL_C(i64(t), NSc::TValue(TStringBuf(ToString(i64(t)))).ForceIntNumber(), ToString(t)); - UNIT_ASSERT_VALUES_EQUAL_C(ToString(double(t)), ToString(NSc::TValue(TStringBuf(ToString(double(t)))).ForceNumber()), ToString(t)); + UNIT_ASSERT_VALUES_EQUAL_C(i64(t), NSc::TValue(TStringBuf(ToString(i64(t)))).ForceIntNumber(), ToString(t)); + UNIT_ASSERT_VALUES_EQUAL_C(ToString(double(t)), ToString(NSc::TValue(TStringBuf(ToString(double(t)))).ForceNumber()), ToString(t)); - UNIT_ASSERT_VALUES_EQUAL_C(ToString(i64(t)), NSc::TValue(TStringBuf(ToString(i64(t)))).ForceString(), ToString(t)); - UNIT_ASSERT_VALUES_EQUAL_C(ToString(double(t)), NSc::TValue(TStringBuf(ToString(double(t)))).ForceString(), ToString(t)); + UNIT_ASSERT_VALUES_EQUAL_C(ToString(i64(t)), NSc::TValue(TStringBuf(ToString(i64(t)))).ForceString(), ToString(t)); + UNIT_ASSERT_VALUES_EQUAL_C(ToString(double(t)), NSc::TValue(TStringBuf(ToString(double(t)))).ForceString(), ToString(t)); } Y_UNIT_TEST(TestForce) { @@ -258,8 +258,8 @@ Y_UNIT_TEST_SUITE(TSchemeTest) { DoCheckAssignmentNum<double>(i, iless, imore, ii, "double"); } - // DoCheckAssignment<bool>(true, true, true, "true", "bool"); - // DoCheckAssignment<bool>(false, false, false, "false", "bool"); + // DoCheckAssignment<bool>(true, true, true, "true", "bool"); + // DoCheckAssignment<bool>(false, false, false, "false", "bool"); for (int i = 1; i < 3; ++i) { TString ii = ToString(i); @@ -299,12 +299,12 @@ Y_UNIT_TEST_SUITE(TSchemeTest) { DoCheckAssignment<const char*>("www", "wwa", "wwz", "\"www\"", "const char*"); DoCheckAssignmentArr("xxx", "xxa", "xxz", "\"xxx\"", "const char[]"); DoCheckAssignment<TStringBuf>("yyy", "yya", "yyz", "\"yyy\"", "TStringBuf"); - -#if defined(_MSC_VER) - //TODO -#else + +#if defined(_MSC_VER) + //TODO +#else DoCheckAssignment<TString>("ttt", "tta", "ttz", "\"ttt\"", "TString"); -#endif +#endif NSc::TValue v; v.SetDict(); @@ -351,10 +351,10 @@ Y_UNIT_TEST_SUITE(TSchemeTest) { v.Insert(2, "c"); UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d"])"); - v.AppendAll({1, 2, 3}); + v.AppendAll({1, 2, 3}); UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d",1,2,3])"); - TVector<int> d{4, 5, 6}; + TVector<int> d{4, 5, 6}; v.AppendAll(d); UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"(["a","b","c","d",1,2,3,4,5,6])"); UNIT_ASSERT_VALUES_EQUAL(d.size(), 3u); @@ -365,7 +365,7 @@ Y_UNIT_TEST_SUITE(TSchemeTest) { UNIT_ASSERT_VALUES_EQUAL(v.Clone().Clear().AppendAll(s).ToJson(), R"(["x","y","z"])"); UNIT_ASSERT_VALUES_EQUAL(v.Clone().Clear().AppendAll(TVector<TStringBuf>()).ToJson(), R"([])"); - v.AddAll({{"a", "b"}, {"c", "d"}}); + v.AddAll({{"a", "b"}, {"c", "d"}}); UNIT_ASSERT_VALUES_EQUAL(v.ToJson(), R"({"a":"b","c":"d"})"); } diff --git a/library/cpp/scheme/ut_utils/scheme_ut_utils.cpp b/library/cpp/scheme/ut_utils/scheme_ut_utils.cpp index 0bbdab10e8..9ffd03bfbe 100644 --- a/library/cpp/scheme/ut_utils/scheme_ut_utils.cpp +++ b/library/cpp/scheme/ut_utils/scheme_ut_utils.cpp @@ -12,18 +12,18 @@ namespace NSc { } catch (const TSchemeParseException& e) { TStringStream s; NColorizer::TColors colors; - s << "\n" - << colors.YellowColor() << "Reason:" << colors.OldColor() << "\n" - << e.Reason; - s << "\n" - << colors.YellowColor() << "Where:" << colors.OldColor() << "\n" - << val.SubStr(0, e.Offset) << colors.RedColor() << val.SubStr(e.Offset) << colors.OldColor() << "\n"; + s << "\n" + << colors.YellowColor() << "Reason:" << colors.OldColor() << "\n" + << e.Reason; + s << "\n" + << colors.YellowColor() << "Where:" << colors.OldColor() << "\n" + << val.SubStr(0, e.Offset) << colors.RedColor() << val.SubStr(e.Offset) << colors.OldColor() << "\n"; UNIT_FAIL_IMPL("could not parse json", s.Str()); return NSc::Null(); } catch (const yexception& e) { TStringStream s; - s << '\n' - << val; + s << '\n' + << val; UNIT_FAIL_IMPL("could not parse json", s.Str()); return NSc::Null(); } diff --git a/library/cpp/scheme/ut_utils/scheme_ut_utils.h b/library/cpp/scheme/ut_utils/scheme_ut_utils.h index eb3ea15b2a..351dafe4f8 100644 --- a/library/cpp/scheme/ut_utils/scheme_ut_utils.h +++ b/library/cpp/scheme/ut_utils/scheme_ut_utils.h @@ -24,16 +24,16 @@ namespace NSc { return AssertFromJson(val).ToJson(true); } -#define UNIT_ASSERT_JSON_EQ_JSON_C(A, B, c) \ - do { \ - const TString _a = NSc::NUt::NormalizeJson(A); \ - const TString _b = NSc::NUt::NormalizeJson(B); \ - if (_a != _b) { \ - UNIT_FAIL_IMPL( \ - "json values are different (" #A " != " #B ")", \ +#define UNIT_ASSERT_JSON_EQ_JSON_C(A, B, c) \ + do { \ + const TString _a = NSc::NUt::NormalizeJson(A); \ + const TString _b = NSc::NUt::NormalizeJson(B); \ + if (_a != _b) { \ + UNIT_FAIL_IMPL( \ + "json values are different (" #A " != " #B ")", \ Sprintf("%s\n!=\n%s\n%s\n%s", _a.data(), _b.data(), \ ::NUnitTest::ColoredDiff(NJson::PrettifyJson(_a), NJson::PrettifyJson(_b), " \t\n,:\"{}[]").data(), ToString(c).data())); \ - } \ + } \ } while (false) #define UNIT_ASSERT_JSON_EQ_JSON(A, B) UNIT_ASSERT_JSON_EQ_JSON_C(A, B, "") diff --git a/library/cpp/sighandler/async_signals_handler.cpp b/library/cpp/sighandler/async_signals_handler.cpp index 00ce1c18fb..bf4a176ff3 100644 --- a/library/cpp/sighandler/async_signals_handler.cpp +++ b/library/cpp/sighandler/async_signals_handler.cpp @@ -1,5 +1,5 @@ #include "async_signals_handler.h" - + #include <util/system/platform.h> #if !defined(_win_) @@ -11,10 +11,10 @@ #include <unistd.h> -#if defined(_linux_) -#include <dlfcn.h> -#endif - +#if defined(_linux_) +#include <dlfcn.h> +#endif + #include <util/system/atomic.h> #include <util/system/defaults.h> #include <util/system/event.h> @@ -22,187 +22,187 @@ #include <util/system/spinlock.h> #include <util/system/thread.h> #include <util/system/yassert.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> -namespace { +namespace { volatile int SIGNAL_PIPE_WRITE_FD = 0; // will be initialized in ctor void WriteAllOrDie(const int fd, const void* buf, size_t bufsize) { - size_t totalBytesWritten = 0; - - while (totalBytesWritten != bufsize) { - const ssize_t result = write(fd, (const char*)buf + totalBytesWritten, bufsize - totalBytesWritten); - - Y_VERIFY(result >= 0 || (result == -1 && errno == EINTR), "write failed: %s (errno = %d)", strerror(errno), errno); - totalBytesWritten += static_cast<size_t>(result); - } + size_t totalBytesWritten = 0; + + while (totalBytesWritten != bufsize) { + const ssize_t result = write(fd, (const char*)buf + totalBytesWritten, bufsize - totalBytesWritten); + + Y_VERIFY(result >= 0 || (result == -1 && errno == EINTR), "write failed: %s (errno = %d)", strerror(errno), errno); + totalBytesWritten += static_cast<size_t>(result); + } } void PipeWriterSignalHandler(int, siginfo_t* info, void*) { - const ui8 signum = static_cast<ui8>(info->si_signo); - - WriteAllOrDie(SIGNAL_PIPE_WRITE_FD, &signum, 1); - } - - // Handler for the "asynchronous" unix signals (those which can occur - // at arbitrary point of execution and have no need to be reacted on instantly - // and/or to preserve execution context at the point of interrupt). - // - // Async signals -- SIGHUP, SIGUSR1 (used to cause configuration files reread for example) - // Sync signals -- fatal errors like SIGSEGV, SIGBUS... - class TAsyncSignalsHandler { - private: - TThread Thread; - int SignalPipeReadFd; - typedef TAutoPtr<TEventHandler> TEventHandlerPtr; - THashMap<int, TEventHandlerPtr> Handlers; - TRWMutex HandlersLock; - - TAtomic ShouldDie; + const ui8 signum = static_cast<ui8>(info->si_signo); + + WriteAllOrDie(SIGNAL_PIPE_WRITE_FD, &signum, 1); + } + + // Handler for the "asynchronous" unix signals (those which can occur + // at arbitrary point of execution and have no need to be reacted on instantly + // and/or to preserve execution context at the point of interrupt). + // + // Async signals -- SIGHUP, SIGUSR1 (used to cause configuration files reread for example) + // Sync signals -- fatal errors like SIGSEGV, SIGBUS... + class TAsyncSignalsHandler { + private: + TThread Thread; + int SignalPipeReadFd; + typedef TAutoPtr<TEventHandler> TEventHandlerPtr; + THashMap<int, TEventHandlerPtr> Handlers; + TRWMutex HandlersLock; + + TAtomic ShouldDie; TSystemEvent DieEvent; - static void* ThreadFunc(void* data) { - reinterpret_cast<TAsyncSignalsHandler*>(data)->RealThreadFunc(); - - return nullptr; + static void* ThreadFunc(void* data) { + reinterpret_cast<TAsyncSignalsHandler*>(data)->RealThreadFunc(); + + return nullptr; + } + + inline void RealThreadFunc() { + for (;;) { + ui8 signum; + const ssize_t bytesRead = read(SignalPipeReadFd, &signum, 1); + + Y_VERIFY(bytesRead >= 0 || (bytesRead == -1 && errno == EINTR), "read failed: %s (errno = %d)", strerror(errno), errno); + + if (AtomicAdd(ShouldDie, 0) != 0) { + DieEvent.Signal(); + + break; + } + + if (bytesRead == 0) { + break; + } else if (bytesRead == -1) { + continue; + } + + { + TReadGuard dnd(HandlersLock); + + const TEventHandlerPtr* handler = Handlers.FindPtr(signum); + Y_VERIFY(handler && handler->Get(), "Async signal handler is not set, it's a bug!"); + handler->Get()->Handle(signum); + } + } } - inline void RealThreadFunc() { - for (;;) { - ui8 signum; - const ssize_t bytesRead = read(SignalPipeReadFd, &signum, 1); - - Y_VERIFY(bytesRead >= 0 || (bytesRead == -1 && errno == EINTR), "read failed: %s (errno = %d)", strerror(errno), errno); - - if (AtomicAdd(ShouldDie, 0) != 0) { - DieEvent.Signal(); - - break; - } - - if (bytesRead == 0) { - break; - } else if (bytesRead == -1) { - continue; - } - - { - TReadGuard dnd(HandlersLock); - - const TEventHandlerPtr* handler = Handlers.FindPtr(signum); - Y_VERIFY(handler && handler->Get(), "Async signal handler is not set, it's a bug!"); - handler->Get()->Handle(signum); - } - } - } - - public: - TAsyncSignalsHandler() - : Thread(TThread::TParams(ThreadFunc, this).SetName("sighandler")) - , SignalPipeReadFd(0) - , ShouldDie(0) - { - int filedes[2] = {-1}; + public: + TAsyncSignalsHandler() + : Thread(TThread::TParams(ThreadFunc, this).SetName("sighandler")) + , SignalPipeReadFd(0) + , ShouldDie(0) + { + int filedes[2] = {-1}; #ifdef _linux_ - int result; - - { - using pipe2_t = decltype(pipe2); - pipe2_t* pipe2Ptr = (pipe2_t*)dlsym(RTLD_DEFAULT, "pipe2"); - -#if defined(_musl_) - if (!pipe2Ptr) { - pipe2Ptr = pipe2; - } + int result; + + { + using pipe2_t = decltype(pipe2); + pipe2_t* pipe2Ptr = (pipe2_t*)dlsym(RTLD_DEFAULT, "pipe2"); + +#if defined(_musl_) + if (!pipe2Ptr) { + pipe2Ptr = pipe2; + } +#endif + + if (pipe2Ptr) { + result = pipe2Ptr(filedes, O_CLOEXEC); + } else { + result = -1; + errno = ENOSYS; + } + } + + if (result != 0 && errno == ENOSYS) { // linux older than 2.6.27 returns "not implemented" #endif + Y_VERIFY(pipe(filedes) == 0, "pipe failed: %s (errno = %d)", strerror(errno), errno); - if (pipe2Ptr) { - result = pipe2Ptr(filedes, O_CLOEXEC); - } else { - result = -1; - errno = ENOSYS; - } - } - - if (result != 0 && errno == ENOSYS) { // linux older than 2.6.27 returns "not implemented" -#endif - Y_VERIFY(pipe(filedes) == 0, "pipe failed: %s (errno = %d)", strerror(errno), errno); - - SignalPipeReadFd = filedes[0]; - SIGNAL_PIPE_WRITE_FD = filedes[1]; - - Y_VERIFY(fcntl(SignalPipeReadFd, F_SETFD, FD_CLOEXEC) == 0, "fcntl failed: %s (errno = %d)", strerror(errno), errno); - Y_VERIFY(fcntl(SIGNAL_PIPE_WRITE_FD, F_SETFD, FD_CLOEXEC) == 0, "fcntl failed: %s (errno = %d)", strerror(errno), errno); + SignalPipeReadFd = filedes[0]; + SIGNAL_PIPE_WRITE_FD = filedes[1]; + + Y_VERIFY(fcntl(SignalPipeReadFd, F_SETFD, FD_CLOEXEC) == 0, "fcntl failed: %s (errno = %d)", strerror(errno), errno); + Y_VERIFY(fcntl(SIGNAL_PIPE_WRITE_FD, F_SETFD, FD_CLOEXEC) == 0, "fcntl failed: %s (errno = %d)", strerror(errno), errno); #ifdef _linux_ - } else { - Y_VERIFY(result == 0, "pipe2 failed: %s (errno = %d)", strerror(errno), errno); - SignalPipeReadFd = filedes[0]; - SIGNAL_PIPE_WRITE_FD = filedes[1]; - } + } else { + Y_VERIFY(result == 0, "pipe2 failed: %s (errno = %d)", strerror(errno), errno); + SignalPipeReadFd = filedes[0]; + SIGNAL_PIPE_WRITE_FD = filedes[1]; + } #endif - Thread.Start(); - Thread.Detach(); - } + Thread.Start(); + Thread.Detach(); + } - ~TAsyncSignalsHandler() { - AtomicSwap(&ShouldDie, TAtomic(1)); - ui8 fakeSignal = 0; - WriteAllOrDie(SIGNAL_PIPE_WRITE_FD, &fakeSignal, 1); + ~TAsyncSignalsHandler() { + AtomicSwap(&ShouldDie, TAtomic(1)); + ui8 fakeSignal = 0; + WriteAllOrDie(SIGNAL_PIPE_WRITE_FD, &fakeSignal, 1); - DieEvent.WaitT(TDuration::Seconds(15)); + DieEvent.WaitT(TDuration::Seconds(15)); - /* may cause VERIFY failure in signal handler, propably we should leave it to process clean procedure + /* may cause VERIFY failure in signal handler, propably we should leave it to process clean procedure close(SIGNAL_PIPE_WRITE_FD); close(SignalPipeReadFd); */ + } + + bool DoInstall(int signum, TAutoPtr<TEventHandler> handler) { + TWriteGuard dnd(HandlersLock); + TEventHandlerPtr& ev = Handlers[signum]; + const bool ret = !ev; + + ev = handler; + + return ret; + } + + void Install(int signum, TAutoPtr<TEventHandler> handler) { + if (DoInstall(signum, handler)) { + struct sigaction a; + + memset(&a, 0, sizeof(a)); + a.sa_sigaction = PipeWriterSignalHandler; + a.sa_flags = SA_SIGINFO | SA_RESTART; + + Y_VERIFY(!sigaction(signum, &a, nullptr), "sigaction failed: %s (errno = %d)", strerror(errno), errno); + } } - - bool DoInstall(int signum, TAutoPtr<TEventHandler> handler) { - TWriteGuard dnd(HandlersLock); - TEventHandlerPtr& ev = Handlers[signum]; - const bool ret = !ev; - - ev = handler; - - return ret; - } - - void Install(int signum, TAutoPtr<TEventHandler> handler) { - if (DoInstall(signum, handler)) { - struct sigaction a; - - memset(&a, 0, sizeof(a)); - a.sa_sigaction = PipeWriterSignalHandler; - a.sa_flags = SA_SIGINFO | SA_RESTART; - - Y_VERIFY(!sigaction(signum, &a, nullptr), "sigaction failed: %s (errno = %d)", strerror(errno), errno); - } - } - }; - - // This pointer is never deleted - yeah, it's intended memory leak. - // It is necessary to prevent problems when user's signal handler calls exit function - // which destroys all global variables including this one. - // It such situation we have 2 options: - // - wait for auxiliary thread to die - which will cause dead lock - // - destruct variable, ignoring thread - which will cause data corruption. + }; + + // This pointer is never deleted - yeah, it's intended memory leak. + // It is necessary to prevent problems when user's signal handler calls exit function + // which destroys all global variables including this one. + // It such situation we have 2 options: + // - wait for auxiliary thread to die - which will cause dead lock + // - destruct variable, ignoring thread - which will cause data corruption. TAsyncSignalsHandler* SIGNALS_HANDLER = nullptr; -} +} void SetAsyncSignalHandler(int signum, TAutoPtr<TEventHandler> handler) { static TAtomic lock; - + if (Y_UNLIKELY(SIGNALS_HANDLER == nullptr)) { TGuard<TAtomic> dnd(lock); - + if (SIGNALS_HANDLER == nullptr) { // NEVERS GETS DESTROYED SIGNALS_HANDLER = new TAsyncSignalsHandler(); } } - + SIGNALS_HANDLER->Install(signum, handler); } @@ -218,25 +218,25 @@ namespace { template <typename TFunc> class TFunctionEventHandler: public TEventHandler { TFunc Func; - + public: TFunctionEventHandler(TFunc func) { if (func) Func = func; - } + } int Handle(int signum) override { - if (Func) { - Func(signum); - } - + if (Func) { + Func(signum); + } + return 0; } }; } void SetAsyncSignalHandler(int signum, void (*handler)(int)) { - SetAsyncSignalHandler(signum, new TFunctionEventHandler<void (*)(int)>(handler)); + SetAsyncSignalHandler(signum, new TFunctionEventHandler<void (*)(int)>(handler)); } void SetAsyncSignalFunction(int signum, std::function<void(int)> func) { diff --git a/library/cpp/sighandler/async_signals_handler.h b/library/cpp/sighandler/async_signals_handler.h index da36365ace..8bb44d78a7 100644 --- a/library/cpp/sighandler/async_signals_handler.h +++ b/library/cpp/sighandler/async_signals_handler.h @@ -4,8 +4,8 @@ #include <functional> struct TEventHandler { - virtual ~TEventHandler() { - } + virtual ~TEventHandler() { + } virtual int Handle(int signum) = 0; }; diff --git a/library/cpp/sliding_window/sliding_window.h b/library/cpp/sliding_window/sliding_window.h index 180bdf93d0..2613fd46ea 100644 --- a/library/cpp/sliding_window/sliding_window.h +++ b/library/cpp/sliding_window/sliding_window.h @@ -11,214 +11,214 @@ #include <limits> namespace NSlidingWindow { - namespace NPrivate { - template <class TValueType_, class TCmp, TValueType_ initialValue> // std::less for max, std::greater for min - struct TMinMaxOperationImpl { - using TValueType = TValueType_; - using TValueVector = TVector<TValueType>; - - static constexpr TValueType InitialValue() { - return initialValue; + namespace NPrivate { + template <class TValueType_, class TCmp, TValueType_ initialValue> // std::less for max, std::greater for min + struct TMinMaxOperationImpl { + using TValueType = TValueType_; + using TValueVector = TVector<TValueType>; + + static constexpr TValueType InitialValue() { + return initialValue; + } + + // Updates value in current bucket and returns window value + static TValueType UpdateBucket(TValueType windowValue, TValueVector& buckets, size_t index, TValueType newVal) { + Y_ASSERT(index < buckets.size()); + TCmp cmp; + TValueType& curVal = buckets[index]; + if (cmp(curVal, newVal)) { + curVal = newVal; + if (cmp(windowValue, newVal)) { + windowValue = newVal; + } + } + return windowValue; } - // Updates value in current bucket and returns window value - static TValueType UpdateBucket(TValueType windowValue, TValueVector& buckets, size_t index, TValueType newVal) { - Y_ASSERT(index < buckets.size()); - TCmp cmp; - TValueType& curVal = buckets[index]; - if (cmp(curVal, newVal)) { - curVal = newVal; - if (cmp(windowValue, newVal)) { - windowValue = newVal; - } + static TValueType ClearBuckets(TValueType windowValue, TValueVector& buckets, const size_t firstElemIndex, const size_t bucketsToClear) { + Y_ASSERT(!buckets.empty()); + Y_ASSERT(firstElemIndex < buckets.size()); + Y_ASSERT(bucketsToClear <= buckets.size()); + TCmp cmp; + + bool needRecalc = false; + size_t current = firstElemIndex; + const size_t arraySize = buckets.size(); + for (size_t i = 0; i < bucketsToClear; ++i) { + TValueType& curVal = buckets[current]; + if (!needRecalc && windowValue == curVal) { + needRecalc = true; + } + curVal = InitialValue(); + current = (current + 1) % arraySize; } - return windowValue; - } - - static TValueType ClearBuckets(TValueType windowValue, TValueVector& buckets, const size_t firstElemIndex, const size_t bucketsToClear) { - Y_ASSERT(!buckets.empty()); - Y_ASSERT(firstElemIndex < buckets.size()); - Y_ASSERT(bucketsToClear <= buckets.size()); - TCmp cmp; - - bool needRecalc = false; - size_t current = firstElemIndex; - const size_t arraySize = buckets.size(); - for (size_t i = 0; i < bucketsToClear; ++i) { - TValueType& curVal = buckets[current]; - if (!needRecalc && windowValue == curVal) { - needRecalc = true; - } - curVal = InitialValue(); - current = (current + 1) % arraySize; - } - if (needRecalc) { - windowValue = InitialValue(); - for (size_t i = 0; i < firstElemIndex; ++i) { - const TValueType val = buckets[i]; - if (cmp(windowValue, val)) { - windowValue = val; - } - } - for (size_t i = current, size = buckets.size(); i < size; ++i) { - const TValueType val = buckets[i]; - if (cmp(windowValue, val)) { - windowValue = val; - } - } + if (needRecalc) { + windowValue = InitialValue(); + for (size_t i = 0; i < firstElemIndex; ++i) { + const TValueType val = buckets[i]; + if (cmp(windowValue, val)) { + windowValue = val; + } + } + for (size_t i = current, size = buckets.size(); i < size; ++i) { + const TValueType val = buckets[i]; + if (cmp(windowValue, val)) { + windowValue = val; + } + } } - return windowValue; + return windowValue; } - }; - + }; + } - template <class TValueType> - using TMaxOperation = NPrivate::TMinMaxOperationImpl<TValueType, std::less<TValueType>, std::numeric_limits<TValueType>::min()>; - - template <class TValueType> - using TMinOperation = NPrivate::TMinMaxOperationImpl<TValueType, std::greater<TValueType>, std::numeric_limits<TValueType>::max()>; - - template <class TValueType_> - struct TSumOperation { - using TValueType = TValueType_; - using TValueVector = TVector<TValueType>; - - static constexpr TValueType InitialValue() { - return TValueType(); // zero + template <class TValueType> + using TMaxOperation = NPrivate::TMinMaxOperationImpl<TValueType, std::less<TValueType>, std::numeric_limits<TValueType>::min()>; + + template <class TValueType> + using TMinOperation = NPrivate::TMinMaxOperationImpl<TValueType, std::greater<TValueType>, std::numeric_limits<TValueType>::max()>; + + template <class TValueType_> + struct TSumOperation { + using TValueType = TValueType_; + using TValueVector = TVector<TValueType>; + + static constexpr TValueType InitialValue() { + return TValueType(); // zero + } + + // Updates value in current bucket and returns window value + static TValueType UpdateBucket(TValueType windowValue, TValueVector& buckets, size_t index, TValueType newVal) { + Y_ASSERT(index < buckets.size()); + buckets[index] += newVal; + windowValue += newVal; + return windowValue; + } + + static TValueType ClearBuckets(TValueType windowValue, TValueVector& buckets, size_t firstElemIndex, size_t bucketsToClear) { + Y_ASSERT(!buckets.empty()); + Y_ASSERT(firstElemIndex < buckets.size()); + Y_ASSERT(bucketsToClear <= buckets.size()); + + const size_t arraySize = buckets.size(); + for (size_t i = 0; i < bucketsToClear; ++i) { + TValueType& curVal = buckets[firstElemIndex]; + windowValue -= curVal; + curVal = InitialValue(); + firstElemIndex = (firstElemIndex + 1) % arraySize; + } + return windowValue; } - - // Updates value in current bucket and returns window value - static TValueType UpdateBucket(TValueType windowValue, TValueVector& buckets, size_t index, TValueType newVal) { - Y_ASSERT(index < buckets.size()); - buckets[index] += newVal; - windowValue += newVal; - return windowValue; - } - - static TValueType ClearBuckets(TValueType windowValue, TValueVector& buckets, size_t firstElemIndex, size_t bucketsToClear) { - Y_ASSERT(!buckets.empty()); - Y_ASSERT(firstElemIndex < buckets.size()); - Y_ASSERT(bucketsToClear <= buckets.size()); - - const size_t arraySize = buckets.size(); - for (size_t i = 0; i < bucketsToClear; ++i) { - TValueType& curVal = buckets[firstElemIndex]; - windowValue -= curVal; - curVal = InitialValue(); - firstElemIndex = (firstElemIndex + 1) % arraySize; - } - return windowValue; - } - }; - - ///////////////////////////////////////////////////////////////////////////////////////// - // TSlidingWindow - ///////////////////////////////////////////////////////////////////////////////////////// - template <class TOperation, class TMutexImpl = TFakeMutex> - class TSlidingWindow { - public: - using TValueType = typename TOperation::TValueType; + }; + + ///////////////////////////////////////////////////////////////////////////////////////// + // TSlidingWindow + ///////////////////////////////////////////////////////////////////////////////////////// + template <class TOperation, class TMutexImpl = TFakeMutex> + class TSlidingWindow { + public: + using TValueType = typename TOperation::TValueType; using TValueVector = TVector<TValueType>; using TSizeType = typename TValueVector::size_type; - public: + public: TSlidingWindow(const TDuration& length, TSizeType partsNum) - : Mutex() - , Buckets(partsNum, TOperation::InitialValue()) // vector of size partsNum initialized with initial value - , WindowValue(TOperation::InitialValue()) - , FirstElem(0) - , PeriodStart() - , Length(length) - , MicroSecondsPerBucket(length.MicroSeconds() / partsNum) - { + : Mutex() + , Buckets(partsNum, TOperation::InitialValue()) // vector of size partsNum initialized with initial value + , WindowValue(TOperation::InitialValue()) + , FirstElem(0) + , PeriodStart() + , Length(length) + , MicroSecondsPerBucket(length.MicroSeconds() / partsNum) + { + } + + TSlidingWindow(const TSlidingWindow& w) + : Mutex() + { + TGuard<TMutexImpl> guard(&w.Mutex); + Buckets = w.Buckets; + WindowValue = w.WindowValue; + FirstElem = w.FirstElem; + PeriodStart = w.PeriodStart; + Length = w.Length; + MicroSecondsPerBucket = w.MicroSecondsPerBucket; + } + + TSlidingWindow(TSlidingWindow&&) = default; + + TSlidingWindow& operator=(TSlidingWindow&&) = default; + TSlidingWindow& operator=(const TSlidingWindow&) = delete; + + // Period of time + const TDuration& GetDuration() const { + return Length; + } + + // Update window with new value and time + TValueType Update(TValueType val, TInstant t) { + TGuard<TMutexImpl> guard(&Mutex); + AdvanceTime(t); + UpdateCurrentBucket(val); + return WindowValue; + } + + // Update just time, without new values + TValueType Update(TInstant t) { + TGuard<TMutexImpl> guard(&Mutex); + AdvanceTime(t); + return WindowValue; + } + + // Get current window value (without updating current time) + TValueType GetValue() const { + TGuard<TMutexImpl> guard(&Mutex); + return WindowValue; } - TSlidingWindow(const TSlidingWindow& w) - : Mutex() - { - TGuard<TMutexImpl> guard(&w.Mutex); - Buckets = w.Buckets; - WindowValue = w.WindowValue; - FirstElem = w.FirstElem; - PeriodStart = w.PeriodStart; - Length = w.Length; - MicroSecondsPerBucket = w.MicroSecondsPerBucket; - } - - TSlidingWindow(TSlidingWindow&&) = default; - - TSlidingWindow& operator=(TSlidingWindow&&) = default; - TSlidingWindow& operator=(const TSlidingWindow&) = delete; - - // Period of time - const TDuration& GetDuration() const { - return Length; + private: + void UpdateCurrentBucket(TValueType val) { + const TSizeType arraySize = Buckets.size(); + const TSizeType pos = (FirstElem + arraySize - 1) % arraySize; + WindowValue = TOperation::UpdateBucket(WindowValue, Buckets, pos, val); } - // Update window with new value and time - TValueType Update(TValueType val, TInstant t) { - TGuard<TMutexImpl> guard(&Mutex); - AdvanceTime(t); - UpdateCurrentBucket(val); - return WindowValue; - } - - // Update just time, without new values - TValueType Update(TInstant t) { - TGuard<TMutexImpl> guard(&Mutex); - AdvanceTime(t); - return WindowValue; - } - - // Get current window value (without updating current time) - TValueType GetValue() const { - TGuard<TMutexImpl> guard(&Mutex); - return WindowValue; - } - - private: - void UpdateCurrentBucket(TValueType val) { - const TSizeType arraySize = Buckets.size(); - const TSizeType pos = (FirstElem + arraySize - 1) % arraySize; - WindowValue = TOperation::UpdateBucket(WindowValue, Buckets, pos, val); - } - - void AdvanceTime(const TInstant& time) { - if (time < PeriodStart + Length) { - return; - } - - if (PeriodStart.MicroSeconds() == 0) { - PeriodStart = time - Length; - return; - } - - const TInstant& newPeriodStart = time - Length; - const ui64 tmDiff = (newPeriodStart - PeriodStart).MicroSeconds(); - const TSizeType bucketsDiff = tmDiff / MicroSecondsPerBucket; - const TSizeType arraySize = Buckets.size(); - const TSizeType buckets = Min(bucketsDiff, arraySize); - - WindowValue = TOperation::ClearBuckets(WindowValue, Buckets, FirstElem, buckets); - FirstElem = (FirstElem + buckets) % arraySize; - PeriodStart += TDuration::MicroSeconds(bucketsDiff * MicroSecondsPerBucket); - - // Check that PeriodStart lags behind newPeriodStart - // (which is actual, uptodate, precise and equal to time - Length) not more - // then MicroSecondsPerBucket - Y_ASSERT(newPeriodStart >= PeriodStart); - Y_ASSERT((newPeriodStart - PeriodStart).MicroSeconds() <= MicroSecondsPerBucket); - } - - - mutable TMutexImpl Mutex; - TValueVector Buckets; - TValueType WindowValue; - TSizeType FirstElem; - TInstant PeriodStart; - TDuration Length; - ui64 MicroSecondsPerBucket; - }; - -} + void AdvanceTime(const TInstant& time) { + if (time < PeriodStart + Length) { + return; + } + + if (PeriodStart.MicroSeconds() == 0) { + PeriodStart = time - Length; + return; + } + + const TInstant& newPeriodStart = time - Length; + const ui64 tmDiff = (newPeriodStart - PeriodStart).MicroSeconds(); + const TSizeType bucketsDiff = tmDiff / MicroSecondsPerBucket; + const TSizeType arraySize = Buckets.size(); + const TSizeType buckets = Min(bucketsDiff, arraySize); + + WindowValue = TOperation::ClearBuckets(WindowValue, Buckets, FirstElem, buckets); + FirstElem = (FirstElem + buckets) % arraySize; + PeriodStart += TDuration::MicroSeconds(bucketsDiff * MicroSecondsPerBucket); + + // Check that PeriodStart lags behind newPeriodStart + // (which is actual, uptodate, precise and equal to time - Length) not more + // then MicroSecondsPerBucket + Y_ASSERT(newPeriodStart >= PeriodStart); + Y_ASSERT((newPeriodStart - PeriodStart).MicroSeconds() <= MicroSecondsPerBucket); + } + + + mutable TMutexImpl Mutex; + TValueVector Buckets; + TValueType WindowValue; + TSizeType FirstElem; + TInstant PeriodStart; + TDuration Length; + ui64 MicroSecondsPerBucket; + }; + +} diff --git a/library/cpp/sliding_window/sliding_window_ut.cpp b/library/cpp/sliding_window/sliding_window_ut.cpp index 1e7343a8d3..35f04b21b2 100644 --- a/library/cpp/sliding_window/sliding_window_ut.cpp +++ b/library/cpp/sliding_window/sliding_window_ut.cpp @@ -9,40 +9,40 @@ Y_UNIT_TEST_SUITE(TSlidingWindowTest) { TSlidingWindow<TMaxOperation<unsigned>> w(TDuration::Minutes(5), 5); TInstant start = TInstant::MicroSeconds(TDuration::Hours(1).MicroSeconds()); TInstant now = start; - w.Update(5, start); // ~ ~ ~ ~ 5 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ + w.Update(5, start); // ~ ~ ~ ~ 5 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ now += TDuration::Minutes(1) + TDuration::Seconds(1); - w.Update(5, now); // 5 ~ ~ ~ 5 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ + w.Update(5, now); // 5 ~ ~ ~ 5 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ now += TDuration::Minutes(1); - w.Update(3, now); // 5 3 ~ ~ 5 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ + w.Update(3, now); // 5 3 ~ ~ 5 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ now += TDuration::Minutes(3); - w.Update(2, now); // 5 3 ~ ~ 2 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ + w.Update(2, now); // 5 3 ~ ~ 2 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ now += TDuration::Minutes(1); - w.Update(2, now); // 2 3 ~ ~ 2 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 3); // ^ + w.Update(2, now); // 2 3 ~ ~ 2 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 3); // ^ now += TDuration::Minutes(1); - w.Update(2, now); // 2 2 ~ ~ 2 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 2); // ^ + w.Update(2, now); // 2 2 ~ ~ 2 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 2); // ^ now += TDuration::Minutes(5); - w.Update(1, now); // ~ 1 ~ ~ ~ - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 1); // ^ + w.Update(1, now); // ~ 1 ~ ~ ~ + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 1); // ^ // update current bucket - w.Update(2, now); // ~ 2 ~ ~ ~ - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 2); // ^ + w.Update(2, now); // ~ 2 ~ ~ ~ + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 2); // ^ - w.Update(1, now + TDuration::Seconds(30)); // ~ 2 ~ ~ ~ - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 2); // ^ + w.Update(1, now + TDuration::Seconds(30)); // ~ 2 ~ ~ ~ + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 2); // ^ // test idle now += TDuration::Minutes(1); - w.Update(now); // ~ 2 ~ ~ ~ - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 2); // ^ + w.Update(now); // ~ 2 ~ ~ ~ + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 2); // ^ - now += TDuration::Minutes(5); // ~ ~ ~ ~ ~ + now += TDuration::Minutes(5); // ~ ~ ~ ~ ~ UNIT_ASSERT_VALUES_EQUAL(w.Update(now), 0); } @@ -50,40 +50,40 @@ Y_UNIT_TEST_SUITE(TSlidingWindowTest) { TSlidingWindow<TMinOperation<unsigned>> w(TDuration::Minutes(5), 5); TInstant start = TInstant::MicroSeconds(TDuration::Hours(1).MicroSeconds()); TInstant now = start; - w.Update(5, start); // ~ ~ ~ ~ 5 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ + w.Update(5, start); // ~ ~ ~ ~ 5 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ now += TDuration::Minutes(1) + TDuration::Seconds(1); - w.Update(5, now); // 5 ~ ~ ~ 5 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ + w.Update(5, now); // 5 ~ ~ ~ 5 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ now += TDuration::Minutes(1); - w.Update(7, now); // 5 7 ~ ~ 5 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ + w.Update(7, now); // 5 7 ~ ~ 5 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ now += TDuration::Minutes(3); - w.Update(8, now); // 5 7 ~ ~ 8 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ + w.Update(8, now); // 5 7 ~ ~ 8 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ now += TDuration::Minutes(1); - w.Update(8, now); // 8 7 ~ ~ 8 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 7); // ^ + w.Update(8, now); // 8 7 ~ ~ 8 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 7); // ^ now += TDuration::Minutes(1); - w.Update(8, now); // 8 8 ~ ~ 8 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 8); // ^ + w.Update(8, now); // 8 8 ~ ~ 8 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 8); // ^ now += TDuration::Minutes(5); - w.Update(6, now); // ~ 6 ~ ~ ~ - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 6); // ^ + w.Update(6, now); // ~ 6 ~ ~ ~ + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 6); // ^ // update current bucket - w.Update(5, now); // ~ 5 ~ ~ ~ - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ + w.Update(5, now); // ~ 5 ~ ~ ~ + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ - w.Update(6, now + TDuration::Seconds(30)); // ~ 5 ~ ~ ~ - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ + w.Update(6, now + TDuration::Seconds(30)); // ~ 5 ~ ~ ~ + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ // test idle now += TDuration::Minutes(1); - w.Update(now); // ~ 5 ~ ~ ~ - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ + w.Update(now); // ~ 5 ~ ~ ~ + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ - now += TDuration::Minutes(5); // ~ ~ ~ ~ ~ + now += TDuration::Minutes(5); // ~ ~ ~ ~ ~ UNIT_ASSERT_VALUES_EQUAL(w.Update(now), std::numeric_limits<unsigned>::max()); } @@ -93,40 +93,40 @@ Y_UNIT_TEST_SUITE(TSlidingWindowTest) { TInstant start = TInstant::MicroSeconds(TDuration::Hours(1).MicroSeconds()); TInstant now = start; - w.Update(5, start); // 0 0 0 0 5 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ + w.Update(5, start); // 0 0 0 0 5 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 5); // ^ now += TDuration::Minutes(1) + TDuration::Seconds(1); - w.Update(5, now); // 5 0 0 0 5 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 10); // ^ + w.Update(5, now); // 5 0 0 0 5 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 10); // ^ now += TDuration::Minutes(1); - w.Update(3, now); // 5 3 0 0 5 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 13); // ^ + w.Update(3, now); // 5 3 0 0 5 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 13); // ^ now += TDuration::Minutes(3); - w.Update(2, now); // 5 3 0 0 2 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 10); // ^ + w.Update(2, now); // 5 3 0 0 2 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 10); // ^ now += TDuration::Minutes(1); - w.Update(2, now); // 2 3 0 0 2 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 7); // ^ + w.Update(2, now); // 2 3 0 0 2 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 7); // ^ now += TDuration::Minutes(1); - w.Update(2, now); // 2 2 0 0 2 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 6); // ^ + w.Update(2, now); // 2 2 0 0 2 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 6); // ^ now += TDuration::Minutes(5); - w.Update(1, now); // 0 1 0 0 0 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 1); // ^ + w.Update(1, now); // 0 1 0 0 0 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 1); // ^ // update current bucket - w.Update(2, now); // 0 3 0 0 0 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 3); // ^ + w.Update(2, now); // 0 3 0 0 0 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 3); // ^ - w.Update(1, now + TDuration::Seconds(30)); // 0 4 0 0 0 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 4); // ^ + w.Update(1, now + TDuration::Seconds(30)); // 0 4 0 0 0 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 4); // ^ // test idle now += TDuration::Minutes(1); - w.Update(now); // 0 4 0 0 0 - UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 4); // ^ + w.Update(now); // 0 4 0 0 0 + UNIT_ASSERT_VALUES_EQUAL(w.GetValue(), 4); // ^ - now += TDuration::Minutes(5); // 0 0 0 0 0 + now += TDuration::Minutes(5); // 0 0 0 0 0 UNIT_ASSERT_VALUES_EQUAL(w.Update(now), 0); } } diff --git a/library/cpp/sse/sse.h b/library/cpp/sse/sse.h index 19bac17de0..af87a4e530 100644 --- a/library/cpp/sse/sse.h +++ b/library/cpp/sse/sse.h @@ -9,7 +9,7 @@ /* Author: Vitaliy Manushkin <agri@yandex-team.ru>, Danila Kutenin <danlark@yandex-team.ru> */ #include <util/system/platform.h> - + #if (defined(_i386_) || defined(_x86_64_)) && defined(_sse_) #include <xmmintrin.h> #include <emmintrin.h> diff --git a/library/cpp/sse/sse2neon.h b/library/cpp/sse/sse2neon.h index 695dbd3041..7e82be954b 100644 --- a/library/cpp/sse/sse2neon.h +++ b/library/cpp/sse/sse2neon.h @@ -12,7 +12,7 @@ /* Author: Vitaliy Manushkin <agri@yandex-team.ru> */ #include <util/system/platform.h> -#include <util/system/compiler.h> +#include <util/system/compiler.h> #include <util/system/types.h> #if !defined(_arm64_) @@ -200,7 +200,7 @@ inline __m128i _mm_srl_epi16(__m128i a, __m128i count) { return res; } - + inline __m128i _mm_srl_epi32(__m128i a, __m128i count) { __m128i res; res.AsUi32x4 = vshlq_u32(a.AsUi32x4, vdupq_n_s32(-count.AsUi32x4[0])); @@ -235,14 +235,14 @@ using _mm_srli_epi64 = TWrapperSingleNegDup<uint64x2_t, decltype(vshlq_u64), vshlq_u64, decltype(vdupq_n_s64), vdupq_n_s64>; - + inline __m128i _mm_sll_epi16(__m128i a, __m128i count) { __m128i res; res.AsUi16x8 = vshlq_u16(a.AsUi16x8, vdupq_n_s16(count.AsUi16x8[0])); return res; } - + inline __m128i _mm_sll_epi32(__m128i a, __m128i count) { __m128i res; res.AsUi32x4 = vshlq_u32(a.AsUi32x4, vdupq_n_s32(count.AsUi32x4[0])); @@ -291,7 +291,7 @@ template <typename TOp, typename TFunc, TFunc* func, typename TArgument = __m128 struct TWrapperDualF : TBaseWrapper<TArgument> { Y_FORCE_INLINE TWrapperDualF(const TArgument& op1, const TArgument& op2) { - TQType<TOp>::As(TBaseWrapper<TArgument>::Value) = (TOp) func(TQType<TOp>::As(op1), TQType<TOp>::As(op2)); + TQType<TOp>::As(TBaseWrapper<TArgument>::Value) = (TOp) func(TQType<TOp>::As(op1), TQType<TOp>::As(op2)); } }; @@ -311,29 +311,29 @@ inline __m128i _mm_madd_epi16(__m128i a, __m128i b) { int32x4_t aHigh; int32x4_t bLow; int32x4_t bHigh; - #ifdef __LITTLE_ENDIAN__ - aLow[0] = a.AsSi16x8[0]; //!< I couldn't find vector instructions to do that. Feel free to fix this code. - aLow[1] = a.AsSi16x8[2]; - aLow[2] = a.AsSi16x8[4]; - aLow[3] = a.AsSi16x8[6]; - - aHigh[0] = a.AsSi16x8[1]; - aHigh[1] = a.AsSi16x8[3]; - aHigh[2] = a.AsSi16x8[5]; - aHigh[3] = a.AsSi16x8[7]; - - bLow[0] = b.AsSi16x8[0]; - bLow[1] = b.AsSi16x8[2]; - bLow[2] = b.AsSi16x8[4]; - bLow[3] = b.AsSi16x8[6]; - - bHigh[0] = b.AsSi16x8[1]; - bHigh[1] = b.AsSi16x8[3]; - bHigh[2] = b.AsSi16x8[5]; - bHigh[3] = b.AsSi16x8[7]; - #else - #error Not implemented yet. Do it yourself. - #endif + #ifdef __LITTLE_ENDIAN__ + aLow[0] = a.AsSi16x8[0]; //!< I couldn't find vector instructions to do that. Feel free to fix this code. + aLow[1] = a.AsSi16x8[2]; + aLow[2] = a.AsSi16x8[4]; + aLow[3] = a.AsSi16x8[6]; + + aHigh[0] = a.AsSi16x8[1]; + aHigh[1] = a.AsSi16x8[3]; + aHigh[2] = a.AsSi16x8[5]; + aHigh[3] = a.AsSi16x8[7]; + + bLow[0] = b.AsSi16x8[0]; + bLow[1] = b.AsSi16x8[2]; + bLow[2] = b.AsSi16x8[4]; + bLow[3] = b.AsSi16x8[6]; + + bHigh[0] = b.AsSi16x8[1]; + bHigh[1] = b.AsSi16x8[3]; + bHigh[2] = b.AsSi16x8[5]; + bHigh[3] = b.AsSi16x8[7]; + #else + #error Not implemented yet. Do it yourself. + #endif const int32x4_t lowMul = vmulq_u32(aLow, bLow); const int32x4_t highMul = vmulq_u32(aHigh, bHigh); @@ -387,13 +387,13 @@ using _mm_cmplt_epi32 = Y_FORCE_INLINE __m128i _mm_load_si128(const __m128i* ptr) { __m128i result; - result.AsUi64x2 = vld1q_u64((const uint64_t*)ptr); + result.AsUi64x2 = vld1q_u64((const uint64_t*)ptr); return result; } Y_FORCE_INLINE __m128i _mm_loadu_si128(const __m128i* ptr) { __m128i result; - result.AsUi64x2 = vld1q_u64((const uint64_t*)ptr); + result.AsUi64x2 = vld1q_u64((const uint64_t*)ptr); return result; } @@ -402,12 +402,12 @@ Y_FORCE_INLINE __m128i _mm_lddqu_si128(const __m128i* ptr) { } Y_FORCE_INLINE void _mm_storeu_si128(__m128i* ptr, const __m128i& op) { - vst1q_u64((uint64_t*)ptr, op.AsUi64x2); + vst1q_u64((uint64_t*)ptr, op.AsUi64x2); } Y_FORCE_INLINE void _mm_store_si128(__m128i* ptr, const __m128i& op) { - vst1q_u64((uint64_t*)ptr, op.AsUi64x2); + vst1q_u64((uint64_t*)ptr, op.AsUi64x2); } template <typename TOp, typename TFunc, TFunc* func, typename... TParams> @@ -443,7 +443,7 @@ struct _mm_setzero_si128 : TBaseWrapper<__m128i> { struct _mm_loadl_epi64 : TBaseWrapper<__m128i> { Y_FORCE_INLINE _mm_loadl_epi64(const __m128i* p) { - uint64x1_t im = vld1_u64((const uint64_t*)p); + uint64x1_t im = vld1_u64((const uint64_t*)p); TQType<uint64x2_t>::As(Value) = vcombine_u64(im, vdup_n_u64(0)); } }; @@ -451,7 +451,7 @@ struct _mm_loadl_epi64 : TBaseWrapper<__m128i> { struct _mm_storel_epi64 : TBaseWrapper<__m128i> { Y_FORCE_INLINE _mm_storel_epi64(__m128i* a, __m128i op) { - vst1_u64((uint64_t*)a, vget_low_u64(op.AsUi64x2)); + vst1_u64((uint64_t*)a, vget_low_u64(op.AsUi64x2)); } }; @@ -474,14 +474,14 @@ _mm_shuffle_epi32(const __m128i& op1, const ShuffleStruct4& op2) { __m128i result; const ui8 xi[4] = { ui8(op2.x[0] * 4), ui8(op2.x[1] * 4), - ui8(op2.x[2] * 4), ui8(op2.x[3] * 4) - }; + ui8(op2.x[2] * 4), ui8(op2.x[3] * 4) + }; const uint8x16_t transform = { ui8(xi[0]), ui8(xi[0] + 1), ui8(xi[0] + 2), ui8(xi[0] + 3), ui8(xi[1]), ui8(xi[1] + 1), ui8(xi[1] + 2), ui8(xi[1] + 3), ui8(xi[2]), ui8(xi[2] + 1), ui8(xi[2] + 2), ui8(xi[2] + 3), - ui8(xi[3]), ui8(xi[3] + 1), ui8(xi[3] + 2), ui8(xi[3] + 3) - }; + ui8(xi[3]), ui8(xi[3] + 1), ui8(xi[3] + 2), ui8(xi[3] + 3) + }; result.AsUi8x16 = vqtbl1q_u8(op1.AsUi8x16, transform); return result; } @@ -503,7 +503,7 @@ _mm_movemask_epi8(const __m128i& op) { template <int imm> struct THelper_mm_srli_si128 : TBaseWrapper<__m128i> { Y_FORCE_INLINE - THelper_mm_srli_si128(const __m128i a) { + THelper_mm_srli_si128(const __m128i a) { const auto zero = vdupq_n_u8(0); TQType<uint8x16_t>::As(Value) = vextq_u8(a.AsUi8x16, zero, imm); } @@ -520,21 +520,21 @@ struct THelper_mm_srli_si128<16> : TBaseWrapper<__m128i> { #define _mm_srli_si128(a, imm) THelper_mm_srli_si128<imm>(a) -template<int imm> +template<int imm> inline uint8x16_t vextq_u8_function(uint8x16_t a, uint8x16_t b) { return vextq_u8(a, b, imm); } -template<> +template<> inline uint8x16_t vextq_u8_function<16>(uint8x16_t /* a */, uint8x16_t b) { return b; } - + template <int imm> struct THelper_mm_slli_si128 : TBaseWrapper<__m128i> { Y_FORCE_INLINE - THelper_mm_slli_si128(const __m128i a) { + THelper_mm_slli_si128(const __m128i a) { auto zero = vdupq_n_u8(0); TQType<uint8x16_t>::As(Value) = vextq_u8_function<16 - imm>(zero, a.AsUi8x16); } @@ -641,7 +641,7 @@ struct TScalarOutWrapper : TBaseWrapper<TOpOut> { } }; -template<int imm> +template<int imm> int extract_epi8_arm(__m128i arg) { return vgetq_lane_u8(arg.AsUi8x16, imm); } @@ -668,7 +668,7 @@ long long extract_epi64_arm(__m128i arg) { #define _mm_extract_ps(op, imm) _mm_extract_epi32(op, imm) static Y_FORCE_INLINE -__m128i _mm_mul_epu32(__m128i op1, __m128i op2) { +__m128i _mm_mul_epu32(__m128i op1, __m128i op2) { __m128i result; uint32x4_t r1 = vuzp1q_u32(op1.AsUi32x4, op2.AsUi32x4); uint32x4_t r2 = vuzp1q_u32(op2.AsUi32x4, op1.AsUi32x4); @@ -886,13 +886,13 @@ _mm_movemask_ps(const __m128& op) { bits = vshlq_u32(bits, shifts); return vaddvq_u32(bits); } - + Y_FORCE_INLINE i64 _mm_cvtsi128_si64(__m128i a) { - return vgetq_lane_s64(a.AsSi64x2, 0); -} + return vgetq_lane_s64(a.AsSi64x2, 0); +} static inline void _mm_pause() { - __asm__ ("YIELD"); + __asm__ ("YIELD"); } static inline __m128 _mm_rsqrt_ps(__m128 a) { @@ -909,7 +909,7 @@ inline __m128 _mm_cmpunord_ps(__m128 a, __m128 b) { __m128 res; res.AsUi32x4 = vorrq_u32( vmvnq_u32(vceqq_f32(a.AsFloat32x4, a.AsFloat32x4)), //!< 0xffffffff for all nans in a. - vmvnq_u32(vceqq_f32(b.AsFloat32x4, b.AsFloat32x4)) //!< 0xffffffff all nans in b. + vmvnq_u32(vceqq_f32(b.AsFloat32x4, b.AsFloat32x4)) //!< 0xffffffff all nans in b. ); return res; } diff --git a/library/cpp/sse/ut/test.cpp b/library/cpp/sse/ut/test.cpp index 33c999d284..cf0aa0060c 100644 --- a/library/cpp/sse/ut/test.cpp +++ b/library/cpp/sse/ut/test.cpp @@ -39,7 +39,7 @@ struct T_mm_CallWrapper { #if defined(_arm64_) #include "library/cpp/sse/sse2neon.h" -#elif defined(_i386_) || defined(_x86_64_) +#elif defined(_i386_) || defined(_x86_64_) #include <xmmintrin.h> #include <emmintrin.h> #include <smmintrin.h> @@ -333,7 +333,7 @@ public: __m128i Value[17]; }; - void Test_mm_byte_shifter(EDirection direction, std::function<TShiftRes (__m128i)> foo); + void Test_mm_byte_shifter(EDirection direction, std::function<TShiftRes (__m128i)> foo); void Test_mm_slli_epi16(); void Test_mm_slli_epi32(); @@ -624,10 +624,10 @@ void TSSEEmulTest::Test_mm_storeu_ps() { } } -template<typename C> +template<typename C> C MakeNumber(unsigned number); -template<> +template<> __m128i MakeNumber<__m128i>(unsigned number) { char data[16] = {0}; memcpy(data, &number, sizeof(number)); @@ -635,7 +635,7 @@ __m128i MakeNumber<__m128i>(unsigned number) { return _mm_loadu_si128((__m128i*)data); } -template<> +template<> unsigned MakeNumber<unsigned>(unsigned number) { return number; } @@ -666,16 +666,16 @@ void TSSEEmulTest::Test_mm_shifter_epiXX() { } } - -void TSSEEmulTest::Test_mm_byte_shifter(EDirection direction, std::function<TShiftRes (__m128i)> foo) { + +void TSSEEmulTest::Test_mm_byte_shifter(EDirection direction, std::function<TShiftRes (__m128i)> foo) { const char data[48] = { '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', - '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00' - }; + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00' + }; const __m128i a = _mm_loadu_si128((__m128i*)(data + 16)); const TShiftRes res = foo(a); @@ -760,7 +760,7 @@ void TSSEEmulTest::Test_mm_slli_epi64() { } void TSSEEmulTest::Test_mm_slli_si128() { - Test_mm_byte_shifter(EDirection::Left, [] (__m128i a) -> TShiftRes { + Test_mm_byte_shifter(EDirection::Left, [] (__m128i a) -> TShiftRes { TShiftRes res; res.Value[0] = _mm_slli_si128(a, 0); res.Value[1] = _mm_slli_si128(a, 1); @@ -881,12 +881,12 @@ void TSSEEmulTest::Test_mm_add_pd() { void TSSEEmulTest::Test_mm_madd_epi16() { alignas(16) const char data1[16] = { '\xAA', '\x00', '\xFF', '\xCC', '\x11', '\x22', '\xBB', '\xAA', - '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C' - }; + '\x33', '\x99', '\x44', '\x88', '\x55', '\x77', '\x66', '\x1C' + }; alignas(16) const char data2[16] = { '\x99', '\x33', '\x1C', '\x55', '\x88', '\x66', '\x77', '\x44', - '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF' - }; + '\x00', '\xAA', '\xAA', '\x11', '\xCC', '\xBB', '\x22', '\xFF' + }; const __m128i value1 = TFuncLoad<__m128i>(&data1); const __m128i value2 = TFuncLoad<__m128i>(&data2); @@ -897,13 +897,13 @@ void TSSEEmulTest::Test_mm_madd_epi16() { for (size_t i = 0; i != 4; ++i) { const size_t dataIdx = i * 2; - const i32 etalonResult = (i32) dataw1[dataIdx] * (i32) dataw2[dataIdx] + (i32) dataw1[dataIdx + 1] * (i32) dataw2[dataIdx + 1]; + const i32 etalonResult = (i32) dataw1[dataIdx] * (i32) dataw2[dataIdx] + (i32) dataw1[dataIdx + 1] * (i32) dataw2[dataIdx + 1]; const i32 value = TQType<int32x4_t>::As(res)[i]; UNIT_ASSERT_EQUAL(value, etalonResult); } } - + template <typename TElem> struct THelperSub { static TElem Call(const TElem op1, const TElem op2) { @@ -1781,8 +1781,8 @@ void TSSEEmulTest::Test_mm_mul_epu32() { __m128i value0 = _mm_loadu_si128((__m128i*)&data0); __m128i value1 = _mm_loadu_si128((__m128i*)&data1); - ui64 mul0 = (ui64) dataw0[0] * (ui64) dataw1[0]; - ui64 mul1 = (ui64) dataw0[2] * (ui64) dataw1[2]; + ui64 mul0 = (ui64) dataw0[0] * (ui64) dataw1[0]; + ui64 mul1 = (ui64) dataw0[2] * (ui64) dataw1[2]; __m128i result = _mm_mul_epu32(value0, value1); @@ -1796,18 +1796,18 @@ void TSSEEmulTest::Test_mm_cmpunord_ps() { alignas(16) char allfs[16] = { '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', - '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff' - }; + '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff' + }; alignas(16) char allzeroes[16] = { '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', - '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00' - }; + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00' + }; const __m128 qnan = _mm_set_ps1(std::numeric_limits<float>::quiet_NaN()); const __m128 snan = _mm_set_ps1(std::numeric_limits<float>::signaling_NaN()); - const __m128 values = _mm_loadu_ps((const float*) valuesBits); - const __m128 values2 = _mm_loadu_ps((const float*) values2Bits); + const __m128 values = _mm_loadu_ps((const float*) valuesBits); + const __m128 values2 = _mm_loadu_ps((const float*) values2Bits); const __m128 mask1 = _mm_cmpunord_ps(qnan, qnan); UNIT_ASSERT_EQUAL(::memcmp(&mask1, &allfs, sizeof(allfs)), 0); @@ -1867,21 +1867,21 @@ void TSSEEmulTest::Test_mm_storeu_pd() { void TSSEEmulTest::Test_mm_andnot_ps() { alignas(16) const char firstBits[16] = { '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff', - '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff' - }; + '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff' + }; alignas(16) const char secondBits[16] = { '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', - '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff' - }; + '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff' + }; alignas(16) const char resBits[16] = { '\x00', '\xff', '\x00', '\x00', '\x00', '\xff', '\x00', '\x00', - '\x00', '\xff', '\x00', '\x00', '\x00', '\xff', '\x00', '\x00' - }; + '\x00', '\xff', '\x00', '\x00', '\x00', '\xff', '\x00', '\x00' + }; - const __m128 value1 = _mm_loadu_ps((const float*) firstBits); - const __m128 value2 = _mm_loadu_ps((const float*) secondBits); + const __m128 value1 = _mm_loadu_ps((const float*) firstBits); + const __m128 value2 = _mm_loadu_ps((const float*) secondBits); const __m128 res = _mm_andnot_ps(value1, value2); UNIT_ASSERT_EQUAL(::memcmp(&res, resBits, sizeof(resBits)), 0); @@ -1971,21 +1971,21 @@ void TSSEEmulTest::Test_mm_loadh_pd() { void TSSEEmulTest::Test_mm_or_ps() { alignas(16) const char bytes1[16] = { '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff', - '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff' - }; + '\x00', '\x00', '\xff', '\xff', '\x00', '\x00', '\xff', '\xff' + }; alignas(16) const char bytes2[16] = { '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', - '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff' - }; + '\x00', '\xff', '\x00', '\xff', '\x00', '\xff', '\x00', '\xff' + }; alignas(16) const char etalon[16] = { '\x00', '\xff', '\xff', '\xff', '\x00', '\xff', '\xff', '\xff', - '\x00', '\xff', '\xff', '\xff', '\x00', '\xff', '\xff', '\xff' - }; + '\x00', '\xff', '\xff', '\xff', '\x00', '\xff', '\xff', '\xff' + }; - const __m128 value1 = _mm_loadu_ps((const float*) bytes1); - const __m128 value2 = _mm_loadu_ps((const float*) bytes2); + const __m128 value1 = _mm_loadu_ps((const float*) bytes1); + const __m128 value2 = _mm_loadu_ps((const float*) bytes2); const __m128 res = _mm_or_ps(value1, value2); UNIT_ASSERT_EQUAL(::memcmp(&res, etalon, sizeof(etalon)), 0); diff --git a/library/cpp/sse/ut/ya.make b/library/cpp/sse/ut/ya.make index 45e104971e..21be851e65 100644 --- a/library/cpp/sse/ut/ya.make +++ b/library/cpp/sse/ut/ya.make @@ -3,7 +3,7 @@ UNITTEST_FOR(library/cpp/sse) OWNER(danlark) SRCS( - test.cpp + test.cpp ) IF (ARCH_X86_64) diff --git a/library/cpp/sse/ya.make b/library/cpp/sse/ya.make index d2351e791d..632bdf98b7 100644 --- a/library/cpp/sse/ya.make +++ b/library/cpp/sse/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - +LIBRARY() + OWNER( g:base danlark ) - -SRCS( + +SRCS( sse.cpp -) - -END() +) + +END() diff --git a/library/cpp/streams/brotli/brotli.cpp b/library/cpp/streams/brotli/brotli.cpp index 38052cb688..ce2567724a 100644 --- a/library/cpp/streams/brotli/brotli.cpp +++ b/library/cpp/streams/brotli/brotli.cpp @@ -122,7 +122,7 @@ void TBrotliCompress::DoFinish() { //////////////////////////////////////////////////////////////////////////////// -class TBrotliDecompress::TImpl: public TAdditionalStorage<TImpl> { +class TBrotliDecompress::TImpl: public TAdditionalStorage<TImpl> { public: TImpl(IInputStream* slave) : Slave_(slave) @@ -190,7 +190,7 @@ private: bool SubstreamFinished_ = false; bool InputExhausted_ = false; - const ui8* InputBuffer_ = nullptr; + const ui8* InputBuffer_ = nullptr; size_t InputAvailable_ = 0; unsigned char* TmpBuf() noexcept { @@ -220,7 +220,7 @@ private: }; TBrotliDecompress::TBrotliDecompress(IInputStream* slave, size_t bufferSize) - : Impl_(new (bufferSize) TImpl(slave)) + : Impl_(new (bufferSize) TImpl(slave)) { } diff --git a/library/cpp/streams/brotli/brotli.h b/library/cpp/streams/brotli/brotli.h index b3af869e29..1d60b41851 100644 --- a/library/cpp/streams/brotli/brotli.h +++ b/library/cpp/streams/brotli/brotli.h @@ -9,7 +9,7 @@ * @{ */ -class TBrotliCompress: public IOutputStream { +class TBrotliCompress: public IOutputStream { public: static constexpr int BEST_QUALITY = 11; @@ -32,7 +32,7 @@ public: //////////////////////////////////////////////////////////////////////////////// -class TBrotliDecompress: public IInputStream { +class TBrotliDecompress: public IInputStream { public: /** @param slave stream to read compressed data from diff --git a/library/cpp/streams/bzip2/bzip2.cpp b/library/cpp/streams/bzip2/bzip2.cpp index bccc5c6807..37c484fed7 100644 --- a/library/cpp/streams/bzip2/bzip2.cpp +++ b/library/cpp/streams/bzip2/bzip2.cpp @@ -1,204 +1,204 @@ -#include "bzip2.h" - -#include <util/memory/addstorage.h> -#include <util/generic/scope.h> - -#include <contrib/libs/libbz2/bzlib.h> - -class TBZipDecompress::TImpl: public TAdditionalStorage<TImpl> { -public: +#include "bzip2.h" + +#include <util/memory/addstorage.h> +#include <util/generic/scope.h> + +#include <contrib/libs/libbz2/bzlib.h> + +class TBZipDecompress::TImpl: public TAdditionalStorage<TImpl> { +public: inline TImpl(IInputStream* input) - : Stream_(input) - { - Zero(BzStream_); - Init(); - } - + : Stream_(input) + { + Zero(BzStream_); + Init(); + } + inline ~TImpl() { - Clear(); - } - - inline void Init() { + Clear(); + } + + inline void Init() { if (BZ2_bzDecompressInit(&BzStream_, 0, 0) != BZ_OK) { - ythrow TBZipDecompressError() << "can not init bzip engine"; - } - } - + ythrow TBZipDecompressError() << "can not init bzip engine"; + } + } + inline void Clear() noexcept { BZ2_bzDecompressEnd(&BzStream_); - } - - inline size_t Read(void* buf, size_t size) { - BzStream_.next_out = (char*)buf; - BzStream_.avail_out = size; - - while (true) { - if (BzStream_.avail_in == 0) { - if (FillInputBuffer() == 0) { - return 0; - } - } - + } + + inline size_t Read(void* buf, size_t size) { + BzStream_.next_out = (char*)buf; + BzStream_.avail_out = size; + + while (true) { + if (BzStream_.avail_in == 0) { + if (FillInputBuffer() == 0) { + return 0; + } + } + switch (BZ2_bzDecompress(&BzStream_)) { - case BZ_STREAM_END: { - Clear(); - Init(); + case BZ_STREAM_END: { + Clear(); + Init(); [[fallthrough]]; - } - - case BZ_OK: { - const size_t processed = size - BzStream_.avail_out; - - if (processed) { - return processed; - } - - break; - } - - default: - ythrow TBZipDecompressError() << "bzip error"; - } - } - } - - inline size_t FillInputBuffer() { - BzStream_.next_in = (char*)AdditionalData(); - BzStream_.avail_in = Stream_->Read(BzStream_.next_in, AdditionalDataLength()); - - return BzStream_.avail_in; - } - -private: + } + + case BZ_OK: { + const size_t processed = size - BzStream_.avail_out; + + if (processed) { + return processed; + } + + break; + } + + default: + ythrow TBZipDecompressError() << "bzip error"; + } + } + } + + inline size_t FillInputBuffer() { + BzStream_.next_in = (char*)AdditionalData(); + BzStream_.avail_in = Stream_->Read(BzStream_.next_in, AdditionalDataLength()); + + return BzStream_.avail_in; + } + +private: IInputStream* Stream_; - bz_stream BzStream_; -}; - + bz_stream BzStream_; +}; + TBZipDecompress::TBZipDecompress(IInputStream* input, size_t bufLen) - : Impl_(new (bufLen) TImpl(input)) -{ -} - + : Impl_(new (bufLen) TImpl(input)) +{ +} + TBZipDecompress::~TBZipDecompress() { -} - -size_t TBZipDecompress::DoRead(void* buf, size_t size) { - return Impl_->Read(buf, size); -} - -class TBZipCompress::TImpl: public TAdditionalStorage<TImpl> { -public: +} + +size_t TBZipDecompress::DoRead(void* buf, size_t size) { + return Impl_->Read(buf, size); +} + +class TBZipCompress::TImpl: public TAdditionalStorage<TImpl> { +public: inline TImpl(IOutputStream* stream, size_t level) - : Stream_(stream) - { - Zero(BzStream_); - + : Stream_(stream) + { + Zero(BzStream_); + if (BZ2_bzCompressInit(&BzStream_, level, 0, 0) != BZ_OK) { - ythrow TBZipCompressError() << "can not init bzip engine"; - } - - BzStream_.next_out = TmpBuf(); - BzStream_.avail_out = TmpBufLen(); - } - + ythrow TBZipCompressError() << "can not init bzip engine"; + } + + BzStream_.next_out = TmpBuf(); + BzStream_.avail_out = TmpBufLen(); + } + inline ~TImpl() { BZ2_bzCompressEnd(&BzStream_); - } - - inline void Write(const void* buf, size_t size) { - BzStream_.next_in = (char*)buf; - BzStream_.avail_in = size; - - Y_DEFER { - BzStream_.next_in = 0; - BzStream_.avail_in = 0; - }; - - while (BzStream_.avail_in) { + } + + inline void Write(const void* buf, size_t size) { + BzStream_.next_in = (char*)buf; + BzStream_.avail_in = size; + + Y_DEFER { + BzStream_.next_in = 0; + BzStream_.avail_in = 0; + }; + + while (BzStream_.avail_in) { const int ret = BZ2_bzCompress(&BzStream_, BZ_RUN); - - switch (ret) { - case BZ_RUN_OK: - continue; - - case BZ_PARAM_ERROR: - case BZ_OUTBUFF_FULL: - Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); - BzStream_.next_out = TmpBuf(); - BzStream_.avail_out = TmpBufLen(); - - break; - - default: - ythrow TBZipCompressError() << "bzip error(" << ret << ", " << BzStream_.avail_out << ")"; - } - } - } - - inline void Flush() { - /* - * TODO ? - */ - } - - inline void Finish() { + + switch (ret) { + case BZ_RUN_OK: + continue; + + case BZ_PARAM_ERROR: + case BZ_OUTBUFF_FULL: + Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); + BzStream_.next_out = TmpBuf(); + BzStream_.avail_out = TmpBufLen(); + + break; + + default: + ythrow TBZipCompressError() << "bzip error(" << ret << ", " << BzStream_.avail_out << ")"; + } + } + } + + inline void Flush() { + /* + * TODO ? + */ + } + + inline void Finish() { int ret = BZ2_bzCompress(&BzStream_, BZ_FINISH); - - while (ret != BZ_STREAM_END) { - Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); - BzStream_.next_out = TmpBuf(); - BzStream_.avail_out = TmpBufLen(); - + + while (ret != BZ_STREAM_END) { + Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); + BzStream_.next_out = TmpBuf(); + BzStream_.avail_out = TmpBufLen(); + ret = BZ2_bzCompress(&BzStream_, BZ_FINISH); - } - - Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); - } - -private: + } + + Stream_->Write(TmpBuf(), TmpBufLen() - BzStream_.avail_out); + } + +private: inline char* TmpBuf() noexcept { - return (char*)AdditionalData(); - } - + return (char*)AdditionalData(); + } + inline size_t TmpBufLen() const noexcept { - return AdditionalDataLength(); - } - -private: + return AdditionalDataLength(); + } + +private: IOutputStream* Stream_; - bz_stream BzStream_; -}; - + bz_stream BzStream_; +}; + TBZipCompress::TBZipCompress(IOutputStream* out, size_t compressionLevel, size_t bufLen) - : Impl_(new (bufLen) TImpl(out, compressionLevel)) -{ -} - + : Impl_(new (bufLen) TImpl(out, compressionLevel)) +{ +} + TBZipCompress::~TBZipCompress() { - try { - Finish(); - } catch (...) { - } -} - -void TBZipCompress::DoWrite(const void* buf, size_t size) { + try { + Finish(); + } catch (...) { + } +} + +void TBZipCompress::DoWrite(const void* buf, size_t size) { if (!Impl_) { - ythrow TBZipCompressError() << "can not write to finished bzip stream"; + ythrow TBZipCompressError() << "can not write to finished bzip stream"; } - Impl_->Write(buf, size); -} - -void TBZipCompress::DoFlush() { + Impl_->Write(buf, size); +} + +void TBZipCompress::DoFlush() { if (Impl_) { Impl_->Flush(); } -} - -void TBZipCompress::DoFinish() { +} + +void TBZipCompress::DoFinish() { THolder<TImpl> impl(Impl_.Release()); if (impl) { impl->Finish(); } -} +} diff --git a/library/cpp/streams/bzip2/bzip2.h b/library/cpp/streams/bzip2/bzip2.h index 2322277ef6..e007ab9b6c 100644 --- a/library/cpp/streams/bzip2/bzip2.h +++ b/library/cpp/streams/bzip2/bzip2.h @@ -1,53 +1,53 @@ #pragma once - -#include <util/stream/input.h> -#include <util/stream/output.h> + +#include <util/stream/input.h> +#include <util/stream/output.h> #include <util/generic/ptr.h> #include <util/generic/yexception.h> - -#define BZIP_BUF_LEN (8 * 1024) -#define BZIP_COMPRESSION_LEVEL 6 - + +#define BZIP_BUF_LEN (8 * 1024) +#define BZIP_COMPRESSION_LEVEL 6 + /** * @addtogroup Streams_Archs * @{ */ -class TBZipException: public yexception { -}; - -class TBZipDecompressError: public TBZipException { -}; - -class TBZipCompressError: public TBZipException { -}; - +class TBZipException: public yexception { +}; + +class TBZipDecompressError: public TBZipException { +}; + +class TBZipCompressError: public TBZipException { +}; + class TBZipDecompress: public IInputStream { -public: +public: TBZipDecompress(IInputStream* input, size_t bufLen = BZIP_BUF_LEN); ~TBZipDecompress() override; - -private: + +private: size_t DoRead(void* buf, size_t size) override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + class TBZipCompress: public IOutputStream { -public: +public: TBZipCompress(IOutputStream* out, size_t compressionLevel = BZIP_COMPRESSION_LEVEL, size_t bufLen = BZIP_BUF_LEN); ~TBZipCompress() override; - -private: + +private: void DoWrite(const void* buf, size_t size) override; void DoFlush() override; void DoFinish() override; - -public: - class TImpl; - THolder<TImpl> Impl_; -}; - + +public: + class TImpl; + THolder<TImpl> Impl_; +}; + /** @} */ diff --git a/library/cpp/streams/bzip2/bzip2_ut.cpp b/library/cpp/streams/bzip2/bzip2_ut.cpp index 69a98f296c..0038735599 100644 --- a/library/cpp/streams/bzip2/bzip2_ut.cpp +++ b/library/cpp/streams/bzip2/bzip2_ut.cpp @@ -1,41 +1,41 @@ -#include "bzip2.h" - +#include "bzip2.h" + #include <library/cpp/testing/unittest/registar.h> - + #include <util/stream/file.h> -#include <util/system/tempfile.h> - -#define ZDATA "./zdata" - +#include <util/system/tempfile.h> + +#define ZDATA "./zdata" + Y_UNIT_TEST_SUITE(TBZipTest) { static const TString data = "8s7d5vc6s5vc67sa4c65ascx6asd4xcv76adsfxv76s"; - + Y_UNIT_TEST(TestCompress) { TUnbufferedFileOutput o(ZDATA); - TBZipCompress c(&o); - + TBZipCompress c(&o); + c.Write(data.data(), data.size()); - c.Finish(); - o.Finish(); - } - + c.Finish(); + o.Finish(); + } + Y_UNIT_TEST(TestDecompress) { - TTempFile tmp(ZDATA); - - { + TTempFile tmp(ZDATA); + + { TUnbufferedFileInput i(ZDATA); - TBZipDecompress d(&i); - - UNIT_ASSERT_EQUAL(d.ReadLine(), data); - } - } - + TBZipDecompress d(&i); + + UNIT_ASSERT_EQUAL(d.ReadLine(), data); + } + } + Y_UNIT_TEST(TestCorrupted) { - TMemoryInput i("blablabla", 10); - TBZipDecompress d(&i); - - UNIT_ASSERT_EXCEPTION(d.ReadLine(), TBZipDecompressError); - } -} - -#undef ZDATA + TMemoryInput i("blablabla", 10); + TBZipDecompress d(&i); + + UNIT_ASSERT_EXCEPTION(d.ReadLine(), TBZipDecompressError); + } +} + +#undef ZDATA diff --git a/library/cpp/streams/bzip2/ut/ya.make b/library/cpp/streams/bzip2/ut/ya.make index 5ef91498ca..4b2b7d1f60 100644 --- a/library/cpp/streams/bzip2/ut/ya.make +++ b/library/cpp/streams/bzip2/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/streams/bzip2) - + OWNER( pg g:util ) - -SRCS( - bzip2_ut.cpp -) - -END() + +SRCS( + bzip2_ut.cpp +) + +END() diff --git a/library/cpp/streams/bzip2/ya.make b/library/cpp/streams/bzip2/ya.make index 122a35837c..9c860f269e 100644 --- a/library/cpp/streams/bzip2/ya.make +++ b/library/cpp/streams/bzip2/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - +LIBRARY() + OWNER( pg g:util ) - -PEERDIR( - contrib/libs/libbz2 -) - -SRCS( - bzip2.cpp -) - -END() + +PEERDIR( + contrib/libs/libbz2 +) + +SRCS( + bzip2.cpp +) + +END() diff --git a/library/cpp/streams/lz/lz.cpp b/library/cpp/streams/lz/lz.cpp index b65bb3ed96..0b0849866a 100644 --- a/library/cpp/streams/lz/lz.cpp +++ b/library/cpp/streams/lz/lz.cpp @@ -1,169 +1,169 @@ -#include "lz.h" - +#include "lz.h" + #include <util/system/yassert.h> -#include <util/system/byteorder.h> -#include <util/memory/addstorage.h> +#include <util/system/byteorder.h> +#include <util/memory/addstorage.h> #include <util/generic/buffer.h> #include <util/generic/utility.h> -#include <util/generic/singleton.h> +#include <util/generic/singleton.h> #include <util/generic/yexception.h> -#include <util/stream/mem.h> - -#include <contrib/libs/lz4/lz4.h> -#include <contrib/libs/fastlz/fastlz.h> -#include <contrib/libs/snappy/snappy.h> -#include <contrib/libs/quicklz/quicklz.h> -#include <contrib/libs/minilzo/minilzo.h> - +#include <util/stream/mem.h> + +#include <contrib/libs/lz4/lz4.h> +#include <contrib/libs/fastlz/fastlz.h> +#include <contrib/libs/snappy/snappy.h> +#include <contrib/libs/quicklz/quicklz.h> +#include <contrib/libs/minilzo/minilzo.h> + static inline ui8 HostToLittle(ui8 t) noexcept { - return t; -} - + return t; +} + static inline ui8 LittleToHost(ui8 t) noexcept { - return t; -} - -struct TCommonData { - static const size_t overhead = sizeof(ui16) + sizeof(ui8); -}; - + return t; +} + +struct TCommonData { + static const size_t overhead = sizeof(ui16) + sizeof(ui8); +}; + const size_t SIGNATURE_SIZE = 4; -template <class TCompressor, class TBase> -class TCompressorBase: public TAdditionalStorage<TCompressorBase<TCompressor, TBase>>, public TCompressor, public TCommonData { -public: +template <class TCompressor, class TBase> +class TCompressorBase: public TAdditionalStorage<TCompressorBase<TCompressor, TBase>>, public TCompressor, public TCommonData { +public: inline TCompressorBase(IOutputStream* slave, ui16 blockSize) - : Slave_(slave) - , BlockSize_(blockSize) - { - /* + : Slave_(slave) + , BlockSize_(blockSize) + { + /* * save signature */ - static_assert(sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE"); - Slave_->Write(TCompressor::signature, sizeof(TCompressor::signature) - 1); + static_assert(sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TCompressor::signature) - 1 == SIGNATURE_SIZE"); + Slave_->Write(TCompressor::signature, sizeof(TCompressor::signature) - 1); - /* + /* * save version */ - this->Save((ui32)1); + this->Save((ui32)1); - /* + /* * save block size */ - this->Save(BlockSize()); - } - + this->Save(BlockSize()); + } + inline ~TCompressorBase() { - } - - inline void Write(const char* buf, size_t len) { - while (len) { - const ui16 toWrite = (ui16)Min<size_t>(len, this->BlockSize()); - - this->WriteBlock(buf, toWrite); - - buf += toWrite; - len -= toWrite; - } - } - - inline void Flush() { - } - - inline void Finish() { - this->Flush(); + } + + inline void Write(const char* buf, size_t len) { + while (len) { + const ui16 toWrite = (ui16)Min<size_t>(len, this->BlockSize()); + + this->WriteBlock(buf, toWrite); + + buf += toWrite; + len -= toWrite; + } + } + + inline void Flush() { + } + + inline void Finish() { + this->Flush(); this->WriteBlock(nullptr, 0); - } - - template <class T> + } + + template <class T> static inline void Save(T t, IOutputStream* out) { - t = HostToLittle(t); - - out->Write(&t, sizeof(t)); - } - - template <class T> - inline void Save(T t) { - Save(t, Slave_); - } - -private: + t = HostToLittle(t); + + out->Write(&t, sizeof(t)); + } + + template <class T> + inline void Save(T t) { + Save(t, Slave_); + } + +private: inline void* Block() const noexcept { - return this->AdditionalData(); - } - + return this->AdditionalData(); + } + inline ui16 BlockSize() const noexcept { - return BlockSize_; - } - - inline void WriteBlock(const void* ptr, ui16 len) { + return BlockSize_; + } + + inline void WriteBlock(const void* ptr, ui16 len) { Y_ASSERT(len <= this->BlockSize()); - - ui8 compressed = false; - - if (len) { + + ui8 compressed = false; + + if (len) { const size_t out = this->Compress((const char*)ptr, len, (char*)Block(), this->AdditionalDataLength()); // catch compressor buffer overrun (e.g. SEARCH-2043) //Y_VERIFY(out <= this->Hint(this->BlockSize())); - - if (out < len || TCompressor::SaveIncompressibleChunks()) { - compressed = true; - ptr = Block(); - len = (ui16)out; - } - } - - char tmp[overhead]; - TMemoryOutput header(tmp, sizeof(tmp)); - - this->Save(len, &header); - this->Save(compressed, &header); - + + if (out < len || TCompressor::SaveIncompressibleChunks()) { + compressed = true; + ptr = Block(); + len = (ui16)out; + } + } + + char tmp[overhead]; + TMemoryOutput header(tmp, sizeof(tmp)); + + this->Save(len, &header); + this->Save(compressed, &header); + using TPart = IOutputStream::TPart; if (ptr) { const TPart parts[] = { TPart(tmp, sizeof(tmp)), TPart(ptr, len), }; - + Slave_->Write(parts, sizeof(parts) / sizeof(*parts)); } else { Slave_->Write(tmp, sizeof(tmp)); } - } - -private: + } + +private: IOutputStream* Slave_; - const ui16 BlockSize_; -}; - -template <class T> + const ui16 BlockSize_; +}; + +template <class T> static inline T GLoad(IInputStream* input) { - T t; - - if (input->Load(&t, sizeof(t)) != sizeof(t)) { - ythrow TDecompressorError() << "stream error"; - } - - return LittleToHost(t); -} - + T t; + + if (input->Load(&t, sizeof(t)) != sizeof(t)) { + ythrow TDecompressorError() << "stream error"; + } + + return LittleToHost(t); +} + class TDecompressSignature { -public: +public: inline TDecompressSignature(IInputStream* input) { - if (input->Load(Buffer_, SIGNATURE_SIZE) != SIGNATURE_SIZE) { - ythrow TDecompressorError() << "can not load stream signature"; + if (input->Load(Buffer_, SIGNATURE_SIZE) != SIGNATURE_SIZE) { + ythrow TDecompressorError() << "can not load stream signature"; } - } + } - template <class TDecompressor> - inline bool Check() const { - static_assert(sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE"); - return memcmp(TDecompressor::signature, Buffer_, SIGNATURE_SIZE) == 0; - } + template <class TDecompressor> + inline bool Check() const { + static_assert(sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE, "expect sizeof(TDecompressor::signature) - 1 == SIGNATURE_SIZE"); + return memcmp(TDecompressor::signature, Buffer_, SIGNATURE_SIZE) == 0; + } -private: - char Buffer_[SIGNATURE_SIZE]; +private: + char Buffer_[SIGNATURE_SIZE]; }; template <class TDecompressor> @@ -177,316 +177,316 @@ static inline IInputStream* ConsumeSignature(IInputStream* input) { template <class TDecompressor> class TDecompressorBaseImpl: public TDecompressor, public TCommonData { -public: - static inline ui32 CheckVer(ui32 v) { - if (v != 1) { +public: + static inline ui32 CheckVer(ui32 v) { + if (v != 1) { ythrow yexception() << TStringBuf("incorrect stream version: ") << v; - } - - return v; - } - + } + + return v; + } + inline TDecompressorBaseImpl(IInputStream* slave) - : Slave_(slave) + : Slave_(slave) , Input_(nullptr, 0) - , Eof_(false) - , Version_(CheckVer(Load<ui32>())) - , BlockSize_(Load<ui16>()) - , OutBufSize_(TDecompressor::Hint(BlockSize_)) + , Eof_(false) + , Version_(CheckVer(Load<ui32>())) + , BlockSize_(Load<ui16>()) + , OutBufSize_(TDecompressor::Hint(BlockSize_)) , Tmp_(2 * OutBufSize_) , In_(Tmp_.Data()) - , Out_(In_ + OutBufSize_) - { - this->InitFromStream(Slave_); - } - + , Out_(In_ + OutBufSize_) + { + this->InitFromStream(Slave_); + } + inline ~TDecompressorBaseImpl() { - } - - inline size_t Read(void* buf, size_t len) { - size_t ret = Input_.Read(buf, len); - - if (ret) { - return ret; - } - - if (Eof_) { - return 0; - } - - this->FillNextBlock(); - - ret = Input_.Read(buf, len); - - if (ret) { - return ret; - } - - Eof_ = true; - - return 0; - } - - inline void FillNextBlock() { - char tmp[overhead]; - - if (Slave_->Load(tmp, sizeof(tmp)) != sizeof(tmp)) { - ythrow TDecompressorError() << "can not read block header"; - } - - TMemoryInput header(tmp, sizeof(tmp)); - - const ui16 len = GLoad<ui16>(&header); + } + + inline size_t Read(void* buf, size_t len) { + size_t ret = Input_.Read(buf, len); + + if (ret) { + return ret; + } + + if (Eof_) { + return 0; + } + + this->FillNextBlock(); + + ret = Input_.Read(buf, len); + + if (ret) { + return ret; + } + + Eof_ = true; + + return 0; + } + + inline void FillNextBlock() { + char tmp[overhead]; + + if (Slave_->Load(tmp, sizeof(tmp)) != sizeof(tmp)) { + ythrow TDecompressorError() << "can not read block header"; + } + + TMemoryInput header(tmp, sizeof(tmp)); + + const ui16 len = GLoad<ui16>(&header); if (len > Tmp_.Capacity()) { ythrow TDecompressorError() << "invalid len inside block header"; } - const ui8 compressed = GLoad<ui8>(&header); - - if (compressed > 1) { - ythrow TDecompressorError() << "broken header"; - } - - if (Slave_->Load(In_, len) != len) { - ythrow TDecompressorError() << "can not read data"; - } - - if (compressed) { - const size_t ret = this->Decompress(In_, len, Out_, OutBufSize_); - - Input_.Reset(Out_, ret); - } else { - Input_.Reset(In_, len); - } - } - - template <class T> - inline T Load() { - return GLoad<T>(Slave_); - } - -protected: + const ui8 compressed = GLoad<ui8>(&header); + + if (compressed > 1) { + ythrow TDecompressorError() << "broken header"; + } + + if (Slave_->Load(In_, len) != len) { + ythrow TDecompressorError() << "can not read data"; + } + + if (compressed) { + const size_t ret = this->Decompress(In_, len, Out_, OutBufSize_); + + Input_.Reset(Out_, ret); + } else { + Input_.Reset(In_, len); + } + } + + template <class T> + inline T Load() { + return GLoad<T>(Slave_); + } + +protected: IInputStream* Slave_; - TMemoryInput Input_; - bool Eof_; - const ui32 Version_; - const ui16 BlockSize_; - const size_t OutBufSize_; + TMemoryInput Input_; + bool Eof_; + const ui32 Version_; + const ui16 BlockSize_; + const size_t OutBufSize_; TBuffer Tmp_; - char* In_; - char* Out_; -}; - + char* In_; + char* Out_; +}; + template <class TDecompressor, class TBase> class TDecompressorBase: public TDecompressorBaseImpl<TDecompressor> { -public: +public: inline TDecompressorBase(IInputStream* slave) - : TDecompressorBaseImpl<TDecompressor>(ConsumeSignature<TDecompressor>(slave)) - { - } + : TDecompressorBaseImpl<TDecompressor>(ConsumeSignature<TDecompressor>(slave)) + { + } inline ~TDecompressorBase() { - } + } }; -#define DEF_COMPRESSOR_COMMON(rname, name) \ - rname::~rname() { \ - try { \ - Finish(); \ - } catch (...) { \ - } \ - } \ - \ - void rname::DoWrite(const void* buf, size_t len) { \ - if (!Impl_) { \ - ythrow yexception() << "can not write to finalized stream"; \ - } \ - \ - Impl_->Write((const char*)buf, len); \ - } \ - \ - void rname::DoFlush() { \ - if (!Impl_) { \ - ythrow yexception() << "can not flush finalized stream"; \ - } \ - \ - Impl_->Flush(); \ - } \ - \ - void rname::DoFinish() { \ - THolder<TImpl> impl(Impl_.Release()); \ - \ - if (impl) { \ - impl->Finish(); \ - } \ - } - -#define DEF_COMPRESSOR(rname, name) \ - class rname::TImpl: public TCompressorBase<name, TImpl> { \ - public: \ +#define DEF_COMPRESSOR_COMMON(rname, name) \ + rname::~rname() { \ + try { \ + Finish(); \ + } catch (...) { \ + } \ + } \ + \ + void rname::DoWrite(const void* buf, size_t len) { \ + if (!Impl_) { \ + ythrow yexception() << "can not write to finalized stream"; \ + } \ + \ + Impl_->Write((const char*)buf, len); \ + } \ + \ + void rname::DoFlush() { \ + if (!Impl_) { \ + ythrow yexception() << "can not flush finalized stream"; \ + } \ + \ + Impl_->Flush(); \ + } \ + \ + void rname::DoFinish() { \ + THolder<TImpl> impl(Impl_.Release()); \ + \ + if (impl) { \ + impl->Finish(); \ + } \ + } + +#define DEF_COMPRESSOR(rname, name) \ + class rname::TImpl: public TCompressorBase<name, TImpl> { \ + public: \ inline TImpl(IOutputStream* out, ui16 blockSize) \ - : TCompressorBase<name, TImpl>(out, blockSize) { \ - } \ - }; \ - \ + : TCompressorBase<name, TImpl>(out, blockSize) { \ + } \ + }; \ + \ rname::rname(IOutputStream* slave, ui16 blockSize) \ - : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize)) { \ - } \ - \ - DEF_COMPRESSOR_COMMON(rname, name) - -#define DEF_DECOMPRESSOR(rname, name) \ - class rname::TImpl: public TDecompressorBase<name, TImpl> { \ - public: \ + : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize)) { \ + } \ + \ + DEF_COMPRESSOR_COMMON(rname, name) + +#define DEF_DECOMPRESSOR(rname, name) \ + class rname::TImpl: public TDecompressorBase<name, TImpl> { \ + public: \ inline TImpl(IInputStream* in) \ - : TDecompressorBase<name, TImpl>(in) { \ - } \ - }; \ - \ + : TDecompressorBase<name, TImpl>(in) { \ + } \ + }; \ + \ rname::rname(IInputStream* slave) \ - : Impl_(new TImpl(slave)) { \ - } \ - \ - rname::~rname() { \ - } \ - \ - size_t rname::DoRead(void* buf, size_t len) { \ - return Impl_->Read(buf, len); \ - } - -/* - * MiniLzo - */ -class TMiniLzo { - class TInit { - public: - inline TInit() { - if (lzo_init() != LZO_E_OK) { - ythrow yexception() << "can not init lzo engine"; - } - } - }; - -public: - static const char signature[]; - - inline TMiniLzo() { - Singleton<TInit>(); - } - + : Impl_(new TImpl(slave)) { \ + } \ + \ + rname::~rname() { \ + } \ + \ + size_t rname::DoRead(void* buf, size_t len) { \ + return Impl_->Read(buf, len); \ + } + +/* + * MiniLzo + */ +class TMiniLzo { + class TInit { + public: + inline TInit() { + if (lzo_init() != LZO_E_OK) { + ythrow yexception() << "can not init lzo engine"; + } + } + }; + +public: + static const char signature[]; + + inline TMiniLzo() { + Singleton<TInit>(); + } + inline ~TMiniLzo() { - } - + } + static inline size_t Hint(size_t len) noexcept { // see SEARCH-2043 and, e.g. examples at // http://stackoverflow.com/questions/4235019/how-to-get-lzo-to-work-with-a-file-stream return len + (len / 16) + 64 + 3; - } - + } + static inline bool SaveIncompressibleChunks() noexcept { - return false; - } -}; - -const char TMiniLzo::signature[] = "YLZO"; - -template <size_t N> -class TFixedArray { -public: + return false; + } +}; + +const char TMiniLzo::signature[] = "YLZO"; + +template <size_t N> +class TFixedArray { +public: inline TFixedArray() noexcept { - memset(WorkMem_, 0, sizeof(WorkMem_)); - } - -protected: - char WorkMem_[N]; -}; - -class TMiniLzoCompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_COMPRESS + 1> { -public: + memset(WorkMem_, 0, sizeof(WorkMem_)); + } + +protected: + char WorkMem_[N]; +}; + +class TMiniLzoCompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_COMPRESS + 1> { +public: inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { - lzo_uint out = 0; - lzo1x_1_compress((const lzo_bytep)data, len, (lzo_bytep)ptr, &out, WorkMem_); - - return out; - } -}; - -class TMiniLzoDecompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_DECOMPRESS + 1> { -public: - inline size_t Decompress(const char* data, size_t len, char* ptr, size_t /*max*/) { - lzo_uint ret = 0; - - lzo1x_decompress((const lzo_bytep)data, len, (lzo_bytep)ptr, &ret, WorkMem_); - - return ret; - } - + lzo_uint out = 0; + lzo1x_1_compress((const lzo_bytep)data, len, (lzo_bytep)ptr, &out, WorkMem_); + + return out; + } +}; + +class TMiniLzoDecompressor: public TMiniLzo, public TFixedArray<LZO1X_MEM_DECOMPRESS + 1> { +public: + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t /*max*/) { + lzo_uint ret = 0; + + lzo1x_decompress((const lzo_bytep)data, len, (lzo_bytep)ptr, &ret, WorkMem_); + + return ret; + } + inline void InitFromStream(IInputStream*) const noexcept { - } -}; - -DEF_COMPRESSOR(TLzoCompress, TMiniLzoCompressor) -DEF_DECOMPRESSOR(TLzoDecompress, TMiniLzoDecompressor) - -/* - * FastLZ - */ -class TFastLZ { -public: - static const char signature[]; - + } +}; + +DEF_COMPRESSOR(TLzoCompress, TMiniLzoCompressor) +DEF_DECOMPRESSOR(TLzoDecompress, TMiniLzoDecompressor) + +/* + * FastLZ + */ +class TFastLZ { +public: + static const char signature[]; + static inline size_t Hint(size_t len) noexcept { - return Max<size_t>((size_t)(len * 1.06), 100); - } - + return Max<size_t>((size_t)(len * 1.06), 100); + } + inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { - return fastlz_compress(data, len, ptr); - } - - inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { - return fastlz_decompress(data, len, ptr, max); - } - + return fastlz_compress(data, len, ptr); + } + + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { + return fastlz_decompress(data, len, ptr, max); + } + inline void InitFromStream(IInputStream*) const noexcept { - } - + } + static inline bool SaveIncompressibleChunks() noexcept { - return false; - } -}; - -const char TFastLZ::signature[] = "YLZF"; - -DEF_COMPRESSOR(TLzfCompress, TFastLZ) -DEF_DECOMPRESSOR(TLzfDecompress, TFastLZ) - -/* + return false; + } +}; + +const char TFastLZ::signature[] = "YLZF"; + +DEF_COMPRESSOR(TLzfCompress, TFastLZ) +DEF_DECOMPRESSOR(TLzfDecompress, TFastLZ) + +/* * LZ4 */ class TLZ4 { -public: - static const char signature[]; +public: + static const char signature[]; static inline size_t Hint(size_t len) noexcept { - return Max<size_t>((size_t)(len * 1.06), 100); - } + return Max<size_t>((size_t)(len * 1.06), 100); + } inline size_t Compress(const char* data, size_t len, char* ptr, size_t dstMaxSize) { return LZ4_compress_default(data, ptr, len, dstMaxSize); - } + } - inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t max) { int res = LZ4_decompress_safe(data, ptr, len, max); - if (res < 0) - ythrow TDecompressorError(); - return res; - } + if (res < 0) + ythrow TDecompressorError(); + return res; + } inline void InitFromStream(IInputStream*) const noexcept { - } + } static inline bool SaveIncompressibleChunks() noexcept { - return false; - } + return false; + } }; const char TLZ4::signature[] = "LZ.4"; @@ -498,32 +498,32 @@ DEF_DECOMPRESSOR(TLz4Decompress, TLZ4) * Snappy */ class TSnappy { -public: - static const char signature[]; +public: + static const char signature[]; static inline size_t Hint(size_t len) noexcept { return Max<size_t>(snappy::MaxCompressedLength(len), 100); - } + } inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { - size_t reslen = 0; - snappy::RawCompress(data, len, ptr, &reslen); - return reslen; - } - - inline size_t Decompress(const char* data, size_t len, char* ptr, size_t) { - size_t srclen = 0; - if (!snappy::GetUncompressedLength(data, len, &srclen) || !snappy::RawUncompress(data, len, ptr)) - ythrow TDecompressorError(); - return srclen; - } + size_t reslen = 0; + snappy::RawCompress(data, len, ptr, &reslen); + return reslen; + } + + inline size_t Decompress(const char* data, size_t len, char* ptr, size_t) { + size_t srclen = 0; + if (!snappy::GetUncompressedLength(data, len, &srclen) || !snappy::RawUncompress(data, len, ptr)) + ythrow TDecompressorError(); + return srclen; + } inline void InitFromStream(IInputStream*) const noexcept { - } + } static inline bool SaveIncompressibleChunks() noexcept { - return false; - } + return false; + } }; const char TSnappy::signature[] = "Snap"; @@ -532,117 +532,117 @@ DEF_COMPRESSOR(TSnappyCompress, TSnappy) DEF_DECOMPRESSOR(TSnappyDecompress, TSnappy) /* - * QuickLZ - */ -class TQuickLZBase { -public: - static const char signature[]; - + * QuickLZ + */ +class TQuickLZBase { +public: + static const char signature[]; + static inline size_t Hint(size_t len) noexcept { - return len + 500; - } - - inline TQuickLZBase() + return len + 500; + } + + inline TQuickLZBase() : Table_(nullptr) - { - } - - inline void Init(unsigned ver, unsigned lev, unsigned mod, unsigned type) { - Table_ = LzqTable(ver, lev, mod); - - if (!Table_) { - ythrow yexception() << "unsupported lzq stream(" << ver << ", " << lev << ", " << mod << ")"; - } - - const size_t size = Table_->Setting(3) + Table_->Setting(type); - - Mem_.Reset(::operator new(size)); - memset(Mem_.Get(), 0, size); - } - + { + } + + inline void Init(unsigned ver, unsigned lev, unsigned mod, unsigned type) { + Table_ = LzqTable(ver, lev, mod); + + if (!Table_) { + ythrow yexception() << "unsupported lzq stream(" << ver << ", " << lev << ", " << mod << ")"; + } + + const size_t size = Table_->Setting(3) + Table_->Setting(type); + + Mem_.Reset(::operator new(size)); + memset(Mem_.Get(), 0, size); + } + inline bool SaveIncompressibleChunks() const noexcept { // we must save incompressible chunks "as is" // after compressor run in streaming mode - return Table_->Setting(3); - } - -protected: - const TQuickLZMethods* Table_; - THolder<void> Mem_; -}; - -const char TQuickLZBase::signature[] = "YLZQ"; - -class TQuickLZCompress: public TQuickLZBase { -public: + return Table_->Setting(3); + } + +protected: + const TQuickLZMethods* Table_; + THolder<void> Mem_; +}; + +const char TQuickLZBase::signature[] = "YLZQ"; + +class TQuickLZCompress: public TQuickLZBase { +public: inline size_t Compress(const char* data, size_t len, char* ptr, size_t /*dstMaxSize*/) { - return Table_->Compress(data, ptr, len, (char*)Mem_.Get()); - } -}; - -class TQuickLZDecompress: public TQuickLZBase { -public: - inline size_t Decompress(const char* data, size_t /*len*/, char* ptr, size_t /*max*/) { - return Table_->Decompress(data, ptr, (char*)Mem_.Get()); - } - + return Table_->Compress(data, ptr, len, (char*)Mem_.Get()); + } +}; + +class TQuickLZDecompress: public TQuickLZBase { +public: + inline size_t Decompress(const char* data, size_t /*len*/, char* ptr, size_t /*max*/) { + return Table_->Decompress(data, ptr, (char*)Mem_.Get()); + } + inline void InitFromStream(IInputStream* in) { - const ui8 ver = ::GLoad<ui8>(in); - const ui8 lev = ::GLoad<ui8>(in); - const ui8 mod = ::GLoad<ui8>(in); - - Init(ver, lev, mod, 2); - } -}; - -class TLzqCompress::TImpl: public TCompressorBase<TQuickLZCompress, TImpl> { -public: + const ui8 ver = ::GLoad<ui8>(in); + const ui8 lev = ::GLoad<ui8>(in); + const ui8 mod = ::GLoad<ui8>(in); + + Init(ver, lev, mod, 2); + } +}; + +class TLzqCompress::TImpl: public TCompressorBase<TQuickLZCompress, TImpl> { +public: inline TImpl(IOutputStream* out, ui16 blockSize, EVersion ver, unsigned level, EMode mode) - : TCompressorBase<TQuickLZCompress, TImpl>(out, blockSize) - { - memset(AdditionalData(), 0, AdditionalDataLength()); - - Init(ver, level, mode, 1); - - Save((ui8)ver); - Save((ui8)level); - Save((ui8)mode); - } -}; - + : TCompressorBase<TQuickLZCompress, TImpl>(out, blockSize) + { + memset(AdditionalData(), 0, AdditionalDataLength()); + + Init(ver, level, mode, 1); + + Save((ui8)ver); + Save((ui8)level); + Save((ui8)mode); + } +}; + TLzqCompress::TLzqCompress(IOutputStream* slave, ui16 blockSize, EVersion ver, unsigned level, EMode mode) - : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize, ver, level, mode)) -{ -} - -DEF_COMPRESSOR_COMMON(TLzqCompress, TQuickLZCompress) -DEF_DECOMPRESSOR(TLzqDecompress, TQuickLZDecompress) - -namespace { - template <class T> - struct TInputHolder { + : Impl_(new (TImpl::Hint(blockSize)) TImpl(slave, blockSize, ver, level, mode)) +{ +} + +DEF_COMPRESSOR_COMMON(TLzqCompress, TQuickLZCompress) +DEF_DECOMPRESSOR(TLzqDecompress, TQuickLZDecompress) + +namespace { + template <class T> + struct TInputHolder { static inline T Set(T t) noexcept { - return t; - } - }; - - template <class T> - struct TInputHolder<TAutoPtr<T>> { + return t; + } + }; + + template <class T> + struct TInputHolder<TAutoPtr<T>> { inline T* Set(TAutoPtr<T> v) noexcept { - V_ = v; - - return V_.Get(); - } - - TAutoPtr<T> V_; - }; - - // Decompressing input streams without signature verification - template <class TInput, class TDecompressor> + V_ = v; + + return V_.Get(); + } + + TAutoPtr<T> V_; + }; + + // Decompressing input streams without signature verification + template <class TInput, class TDecompressor> class TLzDecompressInput: public TInputHolder<TInput>, public IInputStream { public: - inline TLzDecompressInput(TInput in) - : Impl_(this->Set(in)) + inline TLzDecompressInput(TInput in) + : Impl_(this->Set(in)) { } @@ -653,79 +653,79 @@ namespace { private: TDecompressorBaseImpl<TDecompressor> Impl_; - }; -} + }; +} -template <class T> +template <class T> static TAutoPtr<IInputStream> TryOpenLzDecompressorX(const TDecompressSignature& s, T input) { if (s.Check<TLZ4>()) - return new TLzDecompressInput<T, TLZ4>(input); + return new TLzDecompressInput<T, TLZ4>(input); if (s.Check<TSnappy>()) - return new TLzDecompressInput<T, TSnappy>(input); + return new TLzDecompressInput<T, TSnappy>(input); if (s.Check<TMiniLzo>()) - return new TLzDecompressInput<T, TMiniLzoDecompressor>(input); + return new TLzDecompressInput<T, TMiniLzoDecompressor>(input); if (s.Check<TFastLZ>()) - return new TLzDecompressInput<T, TFastLZ>(input); + return new TLzDecompressInput<T, TFastLZ>(input); if (s.Check<TQuickLZDecompress>()) - return new TLzDecompressInput<T, TQuickLZDecompress>(input); + return new TLzDecompressInput<T, TQuickLZDecompress>(input); return nullptr; } -template <class T> +template <class T> static inline TAutoPtr<IInputStream> TryOpenLzDecompressorImpl(const TStringBuf& signature, T input) { if (signature.size() == SIGNATURE_SIZE) { TMemoryInput mem(signature.data(), signature.size()); TDecompressSignature s(&mem); - - return TryOpenLzDecompressorX(s, input); + + return TryOpenLzDecompressorX(s, input); } return nullptr; } -template <class T> +template <class T> static inline TAutoPtr<IInputStream> TryOpenLzDecompressorImpl(T input) { - TDecompressSignature s(&*input); + TDecompressSignature s(&*input); - return TryOpenLzDecompressorX(s, input); -} - -template <class T> + return TryOpenLzDecompressorX(s, input); +} + +template <class T> static inline TAutoPtr<IInputStream> OpenLzDecompressorImpl(T input) { TAutoPtr<IInputStream> ret = TryOpenLzDecompressorImpl(input); - - if (!ret) { + + if (!ret) { ythrow TDecompressorError() << "Unknown compression format"; - } + } return ret; } - + TAutoPtr<IInputStream> OpenLzDecompressor(IInputStream* input) { - return OpenLzDecompressorImpl(input); -} - + return OpenLzDecompressorImpl(input); +} + TAutoPtr<IInputStream> TryOpenLzDecompressor(IInputStream* input) { - return TryOpenLzDecompressorImpl(input); -} - + return TryOpenLzDecompressorImpl(input); +} + TAutoPtr<IInputStream> TryOpenLzDecompressor(const TStringBuf& signature, IInputStream* input) { - return TryOpenLzDecompressorImpl(signature, input); -} - + return TryOpenLzDecompressorImpl(signature, input); +} + TAutoPtr<IInputStream> OpenOwnedLzDecompressor(TAutoPtr<IInputStream> input) { - return OpenLzDecompressorImpl(input); -} - + return OpenLzDecompressorImpl(input); +} + TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(TAutoPtr<IInputStream> input) { - return TryOpenLzDecompressorImpl(input); -} - + return TryOpenLzDecompressorImpl(input); +} + TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(const TStringBuf& signature, TAutoPtr<IInputStream> input) { - return TryOpenLzDecompressorImpl(signature, input); -} + return TryOpenLzDecompressorImpl(signature, input); +} diff --git a/library/cpp/streams/lz/lz.h b/library/cpp/streams/lz/lz.h index 3a2eaad88b..95120fffed 100644 --- a/library/cpp/streams/lz/lz.h +++ b/library/cpp/streams/lz/lz.h @@ -1,7 +1,7 @@ #pragma once - -#include <util/stream/output.h> -#include <util/stream/input.h> + +#include <util/stream/output.h> +#include <util/stream/input.h> #include <util/generic/ptr.h> #include <util/generic/yexception.h> @@ -17,7 +17,7 @@ * for some comparisons. */ -struct TDecompressorError: public yexception { +struct TDecompressorError: public yexception { }; /** @@ -31,18 +31,18 @@ struct TDecompressorError: public yexception { * @see http://code.google.com/p/lz4/ */ class TLz4Compress: public IOutputStream { -public: +public: TLz4Compress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); ~TLz4Compress() override; -private: +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; -private: - class TImpl; - THolder<TImpl> Impl_; +private: + class TImpl; + THolder<TImpl> Impl_; }; /** @@ -51,16 +51,16 @@ private: * @see http://code.google.com/p/lz4/ */ class TLz4Decompress: public IInputStream { -public: +public: TLz4Decompress(IInputStream* slave); ~TLz4Decompress() override; -private: +private: size_t DoRead(void* buf, size_t len) override; -private: - class TImpl; - THolder<TImpl> Impl_; +private: + class TImpl; + THolder<TImpl> Impl_; }; /** @@ -69,18 +69,18 @@ private: * @see http://code.google.com/p/snappy/ */ class TSnappyCompress: public IOutputStream { -public: +public: TSnappyCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); ~TSnappyCompress() override; -private: +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; -private: - class TImpl; - THolder<TImpl> Impl_; +private: + class TImpl; + THolder<TImpl> Impl_; }; /** @@ -89,138 +89,138 @@ private: * @see http://code.google.com/p/snappy/ */ class TSnappyDecompress: public IInputStream { -public: +public: TSnappyDecompress(IInputStream* slave); ~TSnappyDecompress() override; -private: +private: size_t DoRead(void* buf, size_t len) override; -private: - class TImpl; - THolder<TImpl> Impl_; +private: + class TImpl; + THolder<TImpl> Impl_; }; /** * MiniLZO compressing stream. - */ + */ class TLzoCompress: public IOutputStream { -public: +public: TLzoCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); ~TLzoCompress() override; - -private: + +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /** * MiniLZO decompressing stream. */ class TLzoDecompress: public IInputStream { -public: +public: TLzoDecompress(IInputStream* slave); ~TLzoDecompress() override; - -private: + +private: size_t DoRead(void* buf, size_t len) override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /** * FastLZ compressing stream. - */ + */ class TLzfCompress: public IOutputStream { -public: +public: TLzfCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15); ~TLzfCompress() override; - -private: + +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /** * FastLZ decompressing stream. */ class TLzfDecompress: public IInputStream { -public: +public: TLzfDecompress(IInputStream* slave); ~TLzfDecompress() override; - -private: + +private: size_t DoRead(void* buf, size_t len) override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /** * QuickLZ compressing stream. - */ + */ class TLzqCompress: public IOutputStream { -public: - enum EVersion { - V_1_31 = 0, - V_1_40 = 1, - V_1_51 = 2 - }; - - /* +public: + enum EVersion { + V_1_31 = 0, + V_1_40 = 1, + V_1_51 = 2 + }; + + /* * streaming mode - actually, backlog size */ - enum EMode { - M_0 = 0, - M_100000 = 1, - M_1000000 = 2 - }; - + enum EMode { + M_0 = 0, + M_100000 = 1, + M_1000000 = 2 + }; + TLzqCompress(IOutputStream* slave, ui16 maxBlockSize = 1 << 15, - EVersion ver = V_1_31, - unsigned level = 0, - EMode mode = M_0); + EVersion ver = V_1_31, + unsigned level = 0, + EMode mode = M_0); ~TLzqCompress() override; - -private: + +private: void DoWrite(const void* buf, size_t len) override; void DoFlush() override; void DoFinish() override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /** * QuickLZ decompressing stream. */ class TLzqDecompress: public IInputStream { -public: +public: TLzqDecompress(IInputStream* slave); ~TLzqDecompress() override; - -private: + +private: size_t DoRead(void* buf, size_t len) override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + /** @} */ /** @@ -236,7 +236,7 @@ private: TAutoPtr<IInputStream> OpenLzDecompressor(IInputStream* input); TAutoPtr<IInputStream> TryOpenLzDecompressor(IInputStream* input); TAutoPtr<IInputStream> TryOpenLzDecompressor(const TStringBuf& signature, IInputStream* input); - + TAutoPtr<IInputStream> OpenOwnedLzDecompressor(TAutoPtr<IInputStream> input); TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(TAutoPtr<IInputStream> input); TAutoPtr<IInputStream> TryOpenOwnedLzDecompressor(const TStringBuf& signature, TAutoPtr<IInputStream> input); diff --git a/library/cpp/streams/lz/lz_ut.cpp b/library/cpp/streams/lz/lz_ut.cpp index 6876f070fc..54ce318dd4 100644 --- a/library/cpp/streams/lz/lz_ut.cpp +++ b/library/cpp/streams/lz/lz_ut.cpp @@ -1,19 +1,19 @@ -#include "lz.h" - +#include "lz.h" + #include <library/cpp/testing/unittest/registar.h> #include <library/cpp/resource/resource.h> - + #include <util/stream/file.h> #include <util/generic/vector.h> -#include <util/system/tempfile.h> -#include <util/generic/singleton.h> - -#define LDATA "./ldata" +#include <util/system/tempfile.h> +#include <util/generic/singleton.h> + +#define LDATA "./ldata" #define LDATA_RANDOM "./ldata.random" - + static const TString data = "aa aaa aa aaa aa aaa bb bbb bb bbb bb bbb"; - -namespace { + +namespace { /** * Produces well-formed random crap **/ @@ -51,7 +51,7 @@ namespace { for (auto size : sizes) { result.push_back(RandomString(size)); } - result.push_back(NResource::Find("/request.data")); + result.push_back(NResource::Find("/request.data")); return result; } @@ -60,11 +60,11 @@ namespace { } struct TRandomData: public TVector<TString> { - inline TRandomData() { - InitRandomData().swap(*this); - } - }; -} + inline TRandomData() { + InitRandomData().swap(*this); + } + }; +} static const TVector<size_t> bufferSizes = { 127, @@ -73,32 +73,32 @@ static const TVector<size_t> bufferSizes = { }; namespace { - template <TLzqCompress::EVersion Ver, int Level, TLzqCompress::EMode Mode> - struct TLzqCompressX: public TLzqCompress { + template <TLzqCompress::EVersion Ver, int Level, TLzqCompress::EMode Mode> + struct TLzqCompressX: public TLzqCompress { inline TLzqCompressX(IOutputStream* out, size_t bufLen) - : TLzqCompress(out, bufLen, Ver, Level, Mode) - { - } - }; -} - -template <class C> + : TLzqCompress(out, bufLen, Ver, Level, Mode) + { + } + }; +} + +template <class C> static inline void TestGoodDataCompress() { TFixedBufferFileOutput o(LDATA); - C c(&o, 1024); - + C c(&o, 1024); + TString d = data; - - for (size_t i = 0; i < 10; ++i) { + + for (size_t i = 0; i < 10; ++i) { c.Write(d.data(), d.size()); - c << Endl; - d = d + d; - } - - c.Finish(); - o.Finish(); -} - + c << Endl; + d = d + d; + } + + c.Finish(); + o.Finish(); +} + template <class C> static inline void TestIncompressibleDataCompress(const TString& d, size_t bufferSize) { TString testFileName = TestFileName(d, bufferSize); @@ -113,30 +113,30 @@ template <class C> static inline void TestCompress() { TestGoodDataCompress<C>(); for (auto bufferSize : bufferSizes) { - for (auto rd : *Singleton<TRandomData>()) { + for (auto rd : *Singleton<TRandomData>()) { TestIncompressibleDataCompress<C>(rd, bufferSize); } } } -template <class D> +template <class D> static inline void TestGoodDataDecompress() { - TTempFile tmpFile(LDATA); - - { + TTempFile tmpFile(LDATA); + + { TFileInput i1(LDATA); D ld(&i1); - + TString d = data; - + for (size_t i2 = 0; i2 < 10; ++i2) { - UNIT_ASSERT_EQUAL(ld.ReadLine(), d); - - d = d + d; - } - } -} - + UNIT_ASSERT_EQUAL(ld.ReadLine(), d); + + d = d + d; + } + } +} + template <class D> static inline void TestIncompressibleDataDecompress(const TString& d, size_t bufferSize) { TString testFileName = TestFileName(d, bufferSize); @@ -154,25 +154,25 @@ template <class D> static inline void TestDecompress() { TestGoodDataDecompress<D>(); for (auto bufferSize : bufferSizes) { - for (auto rd : *Singleton<TRandomData>()) { + for (auto rd : *Singleton<TRandomData>()) { TestIncompressibleDataDecompress<D>(rd, bufferSize); } } } class TMixedDecompress: public IInputStream { -public: +public: TMixedDecompress(IInputStream* input) - : Slave_(OpenLzDecompressor(input).Release()) - { - } + : Slave_(OpenLzDecompressor(input).Release()) + { + } -private: +private: size_t DoRead(void* buf, size_t len) override { - return Slave_->Read(buf, len); - } + return Slave_->Read(buf, len); + } -private: +private: THolder<IInputStream> Slave_; }; @@ -190,50 +190,50 @@ static inline void TestDecompressError() { Y_UNIT_TEST_SUITE(TLzTest) { Y_UNIT_TEST(TestLzo) { - TestCompress<TLzoCompress>(); - TestDecompress<TLzoDecompress>(); - } - + TestCompress<TLzoCompress>(); + TestDecompress<TLzoDecompress>(); + } + Y_UNIT_TEST(TestLzf) { - TestCompress<TLzfCompress>(); - TestDecompress<TLzfDecompress>(); - } - + TestCompress<TLzfCompress>(); + TestDecompress<TLzfDecompress>(); + } + Y_UNIT_TEST(TestLzq) { - TestCompress<TLzqCompress>(); - TestDecompress<TLzqDecompress>(); - } + TestCompress<TLzqCompress>(); + TestDecompress<TLzqDecompress>(); + } Y_UNIT_TEST(TestLzq151_1) { - TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 1, TLzqCompress::M_0>>(); - TestDecompress<TLzqDecompress>(); - } - + TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 1, TLzqCompress::M_0>>(); + TestDecompress<TLzqDecompress>(); + } + Y_UNIT_TEST(TestLzq151_2) { - TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 2, TLzqCompress::M_100000>>(); - TestDecompress<TLzqDecompress>(); - } - + TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 2, TLzqCompress::M_100000>>(); + TestDecompress<TLzqDecompress>(); + } + Y_UNIT_TEST(TestLzq151_3) { - TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 3, TLzqCompress::M_1000000>>(); - TestDecompress<TLzqDecompress>(); - } - + TestCompress<TLzqCompressX<TLzqCompress::V_1_51, 3, TLzqCompress::M_1000000>>(); + TestDecompress<TLzqDecompress>(); + } + Y_UNIT_TEST(TestLzq140_1) { - TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 1, TLzqCompress::M_0>>(); - TestDecompress<TLzqDecompress>(); - } - + TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 1, TLzqCompress::M_0>>(); + TestDecompress<TLzqDecompress>(); + } + Y_UNIT_TEST(TestLzq140_2) { - TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 2, TLzqCompress::M_100000>>(); - TestDecompress<TLzqDecompress>(); - } - + TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 2, TLzqCompress::M_100000>>(); + TestDecompress<TLzqDecompress>(); + } + Y_UNIT_TEST(TestLzq140_3) { - TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 3, TLzqCompress::M_1000000>>(); - TestDecompress<TLzqDecompress>(); - } - + TestCompress<TLzqCompressX<TLzqCompress::V_1_40, 3, TLzqCompress::M_1000000>>(); + TestDecompress<TLzqDecompress>(); + } + Y_UNIT_TEST(TestLz4) { TestCompress<TLz4Compress>(); TestDecompress<TLz4Decompress>(); @@ -260,21 +260,21 @@ Y_UNIT_TEST_SUITE(TLzTest) { TestDecompressError<TSnappyDecompress, TBufferedOutput>(); TestDecompressError<TMixedDecompress, TBufferedOutput>(); } - + Y_UNIT_TEST(TestFactory) { - TStringStream ss; - - { - TLz4Compress c(&ss); - - c.Write("123456789", 9); - c.Finish(); - } - + TStringStream ss; + + { + TLz4Compress c(&ss); + + c.Write("123456789", 9); + c.Finish(); + } + TAutoPtr<IInputStream> is(OpenOwnedLzDecompressor(new TStringInput(ss.Str()))); - - UNIT_ASSERT_EQUAL(is->ReadAll(), "123456789"); - } + + UNIT_ASSERT_EQUAL(is->ReadAll(), "123456789"); + } Y_UNIT_TEST(TestYQ609) { auto data = NResource::Find("/yq_609.data"); @@ -284,4 +284,4 @@ Y_UNIT_TEST_SUITE(TLzTest) { TLz4Decompress d(&input); UNIT_ASSERT_EXCEPTION(d.ReadAll(), TDecompressorError); } -} +} diff --git a/library/cpp/streams/lz/ut/ya.make b/library/cpp/streams/lz/ut/ya.make index 18288c8ac9..74e5a169c8 100644 --- a/library/cpp/streams/lz/ut/ya.make +++ b/library/cpp/streams/lz/ut/ya.make @@ -1,18 +1,18 @@ UNITTEST_FOR(library/cpp/streams/lz) - + OWNER( pg g:util ) - -RESOURCE( - random.data /random.data - request.data /request.data + +RESOURCE( + random.data /random.data + request.data /request.data yq_609.data /yq_609.data -) - -SRCS( - lz_ut.cpp -) - -END() +) + +SRCS( + lz_ut.cpp +) + +END() diff --git a/library/cpp/streams/lz/ya.make b/library/cpp/streams/lz/ya.make index e5eea0b096..7f83071128 100644 --- a/library/cpp/streams/lz/ya.make +++ b/library/cpp/streams/lz/ya.make @@ -1,20 +1,20 @@ -LIBRARY() - +LIBRARY() + OWNER( pg g:util ) - -PEERDIR( - contrib/libs/fastlz - contrib/libs/lz4 - contrib/libs/minilzo - contrib/libs/quicklz - contrib/libs/snappy -) - -SRCS( - lz.cpp -) - -END() + +PEERDIR( + contrib/libs/fastlz + contrib/libs/lz4 + contrib/libs/minilzo + contrib/libs/quicklz + contrib/libs/snappy +) + +SRCS( + lz.cpp +) + +END() diff --git a/library/cpp/streams/lzma/lzma.cpp b/library/cpp/streams/lzma/lzma.cpp index f1942fa546..f265e0a284 100644 --- a/library/cpp/streams/lzma/lzma.cpp +++ b/library/cpp/streams/lzma/lzma.cpp @@ -1,520 +1,520 @@ -#include "lzma.h" - -#include <util/stream/mem.h> -#include <util/system/context.h> -#include <util/generic/cast.h> -#include <util/memory/addstorage.h> -#include <util/generic/ptr.h> -#include <util/generic/intrlist.h> -#include <util/generic/scope.h> - -extern "C" { -#include <contrib/libs/lzmasdk/LzmaEnc.h> -#include <contrib/libs/lzmasdk/LzmaDec.h> -} - -namespace { - class TMemoryGc { - class TAllocation: public TIntrusiveListItem<TAllocation>, public TAdditionalStorage<TAllocation> { - }; - - public: - inline void* Allocate(size_t len) { - if (len > 1024 * 1024 * 1024) { - return nullptr; - } - - TAllocation* ret = new (len) TAllocation; - - Allocs_.PushBack(ret); - - return ret->AdditionalData(); - } - +#include "lzma.h" + +#include <util/stream/mem.h> +#include <util/system/context.h> +#include <util/generic/cast.h> +#include <util/memory/addstorage.h> +#include <util/generic/ptr.h> +#include <util/generic/intrlist.h> +#include <util/generic/scope.h> + +extern "C" { +#include <contrib/libs/lzmasdk/LzmaEnc.h> +#include <contrib/libs/lzmasdk/LzmaDec.h> +} + +namespace { + class TMemoryGc { + class TAllocation: public TIntrusiveListItem<TAllocation>, public TAdditionalStorage<TAllocation> { + }; + + public: + inline void* Allocate(size_t len) { + if (len > 1024 * 1024 * 1024) { + return nullptr; + } + + TAllocation* ret = new (len) TAllocation; + + Allocs_.PushBack(ret); + + return ret->AdditionalData(); + } + inline void Deallocate(void* ptr) noexcept { - if (ptr) { - delete TAllocation::ObjectFromData(ptr); - } - } - - private: - TIntrusiveListWithAutoDelete<TAllocation, TDelete> Allocs_; - }; - - template <class T> - class TInverseFilter { - class TTrampoLine: public ITrampoLine { - public: - inline TTrampoLine(TInverseFilter* parent) - : Parent_(parent) - { - } - + if (ptr) { + delete TAllocation::ObjectFromData(ptr); + } + } + + private: + TIntrusiveListWithAutoDelete<TAllocation, TDelete> Allocs_; + }; + + template <class T> + class TInverseFilter { + class TTrampoLine: public ITrampoLine { + public: + inline TTrampoLine(TInverseFilter* parent) + : Parent_(parent) + { + } + void DoRun() override { - Parent_->RunFilter(); - } - - private: - TInverseFilter* Parent_; - }; - + Parent_->RunFilter(); + } + + private: + TInverseFilter* Parent_; + }; + class TInput: public IInputStream { - public: - inline TInput(TInverseFilter* parent) - : Parent_(parent) - { - } - + public: + inline TInput(TInverseFilter* parent) + : Parent_(parent) + { + } + ~TInput() override { - } - + } + size_t DoRead(void* ptr, size_t len) override { - return Parent_->ReadImpl(ptr, len); - } - - private: - TInverseFilter* Parent_; - }; - + return Parent_->ReadImpl(ptr, len); + } + + private: + TInverseFilter* Parent_; + }; + class TOutput: public IOutputStream { - public: - inline TOutput(TInverseFilter* parent) - : Parent_(parent) - { - } - + public: + inline TOutput(TInverseFilter* parent) + : Parent_(parent) + { + } + ~TOutput() override { - } - + } + void DoWrite(const void* ptr, size_t len) override { - Parent_->WriteImpl(ptr, len); - } - - private: - TInverseFilter* Parent_; - }; - - public: + Parent_->WriteImpl(ptr, len); + } + + private: + TInverseFilter* Parent_; + }; + + public: inline TInverseFilter(IOutputStream* slave, T* filter) - : Slave_(slave) - , Filter_(filter) - , TrampoLine_(this) - , FilterCtx_(FilterClosure()) - , Finished_(false) + : Slave_(slave) + , Filter_(filter) + , TrampoLine_(this) + , FilterCtx_(FilterClosure()) + , Finished_(false) , In_(nullptr, 0) - { - } - + { + } + virtual ~TInverseFilter() { - if (!UncaughtException()) { - try { - Finish(); - } catch (...) { - } - } else { - //rely on gc - } - } - - inline void Write(const void* ptr, size_t len) { - In_.Reset(ptr, len); - - Y_DEFER { - In_.Reset(0, 0); - }; - - while (In_.Avail()) { - SwitchTo(); - } - } - - inline void Finish() { - if (!Finished_) { - Finished_ = true; - SwitchTo(); - } - } - - private: - inline void RunFilter() { - try { - TInput in(this); - TOutput out(this); - - (*Filter_)(&in, &out); - } catch (...) { - Err_ = std::current_exception(); - } - - SwitchFrom(); - } - - inline TContClosure FilterClosure() { + if (!UncaughtException()) { + try { + Finish(); + } catch (...) { + } + } else { + //rely on gc + } + } + + inline void Write(const void* ptr, size_t len) { + In_.Reset(ptr, len); + + Y_DEFER { + In_.Reset(0, 0); + }; + + while (In_.Avail()) { + SwitchTo(); + } + } + + inline void Finish() { + if (!Finished_) { + Finished_ = true; + SwitchTo(); + } + } + + private: + inline void RunFilter() { + try { + TInput in(this); + TOutput out(this); + + (*Filter_)(&in, &out); + } catch (...) { + Err_ = std::current_exception(); + } + + SwitchFrom(); + } + + inline TContClosure FilterClosure() { return {&TrampoLine_, TArrayRef(Stack_, sizeof(Stack_))}; - } - - inline size_t ReadImpl(void* ptr, size_t len) { - while (!Finished_) { - const size_t ret = In_.Read(ptr, len); - - if (ret) { - return ret; - } - - SwitchFrom(); - } - - return 0; - } - - inline void WriteImpl(const void* ptr, size_t len) { + } + + inline size_t ReadImpl(void* ptr, size_t len) { + while (!Finished_) { + const size_t ret = In_.Read(ptr, len); + + if (ret) { + return ret; + } + + SwitchFrom(); + } + + return 0; + } + + inline void WriteImpl(const void* ptr, size_t len) { Y_ASSERT(!Out_.Avail()); - - Out_.Reset(ptr, len); - - while (Out_.Avail()) { - SwitchFrom(); - } - } - - inline bool FlushImpl() { - if (Out_.Avail()) { + + Out_.Reset(ptr, len); + + while (Out_.Avail()) { + SwitchFrom(); + } + } + + inline bool FlushImpl() { + if (Out_.Avail()) { TransferData(&Out_, Slave_); Out_.Reset(nullptr, 0); - - return true; - } - - return false; - } - - inline void SwitchTo() { - do { - CurrentCtx_.SwitchTo(&FilterCtx_); - - if (Err_) { - Finished_ = true; - - std::rethrow_exception(Err_); - } - } while (FlushImpl()); - } - - inline void SwitchFrom() { - FilterCtx_.SwitchTo(&CurrentCtx_); - } - - private: + + return true; + } + + return false; + } + + inline void SwitchTo() { + do { + CurrentCtx_.SwitchTo(&FilterCtx_); + + if (Err_) { + Finished_ = true; + + std::rethrow_exception(Err_); + } + } while (FlushImpl()); + } + + inline void SwitchFrom() { + FilterCtx_.SwitchTo(&CurrentCtx_); + } + + private: IOutputStream* Slave_; - T* Filter_; - TTrampoLine TrampoLine_; - char Stack_[16 * 1024]; - TContMachineContext FilterCtx_; - TContMachineContext CurrentCtx_; - bool Finished_; - TMemoryInput In_; - TMemoryInput Out_; - std::exception_ptr Err_; - }; - - class TLzma { - public: - class TLzmaInput: public ISeqInStream { - public: + T* Filter_; + TTrampoLine TrampoLine_; + char Stack_[16 * 1024]; + TContMachineContext FilterCtx_; + TContMachineContext CurrentCtx_; + bool Finished_; + TMemoryInput In_; + TMemoryInput Out_; + std::exception_ptr Err_; + }; + + class TLzma { + public: + class TLzmaInput: public ISeqInStream { + public: inline TLzmaInput(IInputStream* slave) - : Slave_(slave) - { - Read = ReadFunc; - } - - private: - static inline SRes ReadFunc(const ISeqInStream* p, void* ptr, size_t* len) { - *len = const_cast<TLzmaInput*>(static_cast<const TLzmaInput*>(p))->Slave_->Read(ptr, *len); - - return SZ_OK; - } - - private: + : Slave_(slave) + { + Read = ReadFunc; + } + + private: + static inline SRes ReadFunc(const ISeqInStream* p, void* ptr, size_t* len) { + *len = const_cast<TLzmaInput*>(static_cast<const TLzmaInput*>(p))->Slave_->Read(ptr, *len); + + return SZ_OK; + } + + private: IInputStream* Slave_; - }; - - class TLzmaOutput: public ISeqOutStream { - public: + }; + + class TLzmaOutput: public ISeqOutStream { + public: inline TLzmaOutput(IOutputStream* slave) - : Slave_(slave) - { - Write = WriteFunc; - } - - private: - static inline size_t WriteFunc(const ISeqOutStream* p, const void* ptr, size_t len) { - const_cast<TLzmaOutput*>(static_cast<const TLzmaOutput*>(p))->Slave_->Write(ptr, len); - - return len; - } - - private: + : Slave_(slave) + { + Write = WriteFunc; + } + + private: + static inline size_t WriteFunc(const ISeqOutStream* p, const void* ptr, size_t len) { + const_cast<TLzmaOutput*>(static_cast<const TLzmaOutput*>(p))->Slave_->Write(ptr, len); + + return len; + } + + private: IOutputStream* Slave_; - }; - - class TAlloc: public ISzAlloc { - public: - inline TAlloc() { - Alloc = AllocFunc; - Free = FreeFunc; - } - - private: - static void* AllocFunc(const ISzAlloc* t, size_t len) { - return static_cast<TAlloc*>(((ISzAlloc*)t))->Gc_.Allocate(len); - } - - static void FreeFunc(const ISzAlloc* t, void* p) { - static_cast<TAlloc*>(((ISzAlloc*)t))->Gc_.Deallocate(p); - } - - private: - TMemoryGc Gc_; - }; - + }; + + class TAlloc: public ISzAlloc { + public: + inline TAlloc() { + Alloc = AllocFunc; + Free = FreeFunc; + } + + private: + static void* AllocFunc(const ISzAlloc* t, size_t len) { + return static_cast<TAlloc*>(((ISzAlloc*)t))->Gc_.Allocate(len); + } + + static void FreeFunc(const ISzAlloc* t, void* p) { + static_cast<TAlloc*>(((ISzAlloc*)t))->Gc_.Deallocate(p); + } + + private: + TMemoryGc Gc_; + }; + inline ISzAlloc* Alloc() noexcept { - return &Alloc_; - } - - static inline void Check(SRes r) { - if (r != SZ_OK) { - ythrow yexception() << "lzma error(" << r << ")"; - } - } - - private: - TAlloc Alloc_; - }; - - class TLzmaCompressBase: public TLzma { - public: - inline TLzmaCompressBase(size_t level) - : H_(LzmaEnc_Create(Alloc())) - { - if (!H_) { - ythrow yexception() << "can not init lzma engine"; - } - - LzmaEncProps_Init(&Props_); - - Props_.level = level; - Props_.dictSize = 0; - Props_.lc = -1; - Props_.lp = -1; - Props_.pb = -1; - Props_.fb = -1; - Props_.numThreads = -1; - Props_.writeEndMark = 1; - - Check(LzmaEnc_SetProps(H_, &Props_)); - size_t bufLen = sizeof(PropsBuf_); - Zero(PropsBuf_); - Check(LzmaEnc_WriteProperties(H_, PropsBuf_, &bufLen)); - } - + return &Alloc_; + } + + static inline void Check(SRes r) { + if (r != SZ_OK) { + ythrow yexception() << "lzma error(" << r << ")"; + } + } + + private: + TAlloc Alloc_; + }; + + class TLzmaCompressBase: public TLzma { + public: + inline TLzmaCompressBase(size_t level) + : H_(LzmaEnc_Create(Alloc())) + { + if (!H_) { + ythrow yexception() << "can not init lzma engine"; + } + + LzmaEncProps_Init(&Props_); + + Props_.level = level; + Props_.dictSize = 0; + Props_.lc = -1; + Props_.lp = -1; + Props_.pb = -1; + Props_.fb = -1; + Props_.numThreads = -1; + Props_.writeEndMark = 1; + + Check(LzmaEnc_SetProps(H_, &Props_)); + size_t bufLen = sizeof(PropsBuf_); + Zero(PropsBuf_); + Check(LzmaEnc_WriteProperties(H_, PropsBuf_, &bufLen)); + } + inline ~TLzmaCompressBase() { - LzmaEnc_Destroy(H_, Alloc(), Alloc()); - } - + LzmaEnc_Destroy(H_, Alloc(), Alloc()); + } + inline void operator()(IInputStream* in, IOutputStream* out) { - TLzmaInput input(in); - TLzmaOutput output(out); - - out->Write(PropsBuf_, sizeof(PropsBuf_)); - + TLzmaInput input(in); + TLzmaOutput output(out); + + out->Write(PropsBuf_, sizeof(PropsBuf_)); + Check(LzmaEnc_Encode(H_, &output, &input, nullptr, Alloc(), Alloc())); - } - - private: - CLzmaEncHandle H_; - CLzmaEncProps Props_; - Byte PropsBuf_[LZMA_PROPS_SIZE]; - }; -} - -class TLzmaCompress::TImpl: public TLzmaCompressBase, public TInverseFilter<TLzmaCompressBase> { -public: + } + + private: + CLzmaEncHandle H_; + CLzmaEncProps Props_; + Byte PropsBuf_[LZMA_PROPS_SIZE]; + }; +} + +class TLzmaCompress::TImpl: public TLzmaCompressBase, public TInverseFilter<TLzmaCompressBase> { +public: inline TImpl(IOutputStream* slave, size_t level) - : TLzmaCompressBase(level) - , TInverseFilter<TLzmaCompressBase>(slave, this) - { - } -}; - -class TLzmaDecompress::TImpl: public TLzma { -public: - inline TImpl() - : InBegin_(nullptr) - , InEnd_(nullptr) - { - LzmaDec_Construct(&H_); - } + : TLzmaCompressBase(level) + , TInverseFilter<TLzmaCompressBase>(slave, this) + { + } +}; + +class TLzmaDecompress::TImpl: public TLzma { +public: + inline TImpl() + : InBegin_(nullptr) + , InEnd_(nullptr) + { + LzmaDec_Construct(&H_); + } inline virtual ~TImpl() { - LzmaDec_Free(&H_, Alloc()); - } - - inline size_t Read(void* ptr, size_t len) { - Byte* pos = (Byte*)ptr; - Byte* end = pos + len; - - retry: - size_t availLen = InEnd_ - InBegin_; - size_t bufLen = end - pos; - ELzmaStatus status; - - Check(LzmaDec_DecodeToBuf(&H_, pos, &bufLen, (Byte*)InBegin_, &availLen, LZMA_FINISH_ANY, &status)); - - InBegin_ += availLen; - pos += bufLen; - - if (status == LZMA_STATUS_NEEDS_MORE_INPUT) { + LzmaDec_Free(&H_, Alloc()); + } + + inline size_t Read(void* ptr, size_t len) { + Byte* pos = (Byte*)ptr; + Byte* end = pos + len; + + retry: + size_t availLen = InEnd_ - InBegin_; + size_t bufLen = end - pos; + ELzmaStatus status; + + Check(LzmaDec_DecodeToBuf(&H_, pos, &bufLen, (Byte*)InBegin_, &availLen, LZMA_FINISH_ANY, &status)); + + InBegin_ += availLen; + pos += bufLen; + + if (status == LZMA_STATUS_NEEDS_MORE_INPUT) { Y_ASSERT(InEnd_ == InBegin_); - if (!Fill()) { - ythrow yexception() << "incomplete lzma stream"; - } - - goto retry; - } - - return pos - (Byte*)ptr; - } - -private: - virtual bool Fill() = 0; - -protected: - CLzmaDec H_; - char* InBegin_; - char* InEnd_; + if (!Fill()) { + ythrow yexception() << "incomplete lzma stream"; + } + + goto retry; + } + + return pos - (Byte*)ptr; + } + +private: + virtual bool Fill() = 0; + +protected: + CLzmaDec H_; + char* InBegin_; + char* InEnd_; }; class TLzmaDecompress::TImplStream: public TImpl { -public: +public: inline TImplStream(IInputStream* slave) - : Slave_(slave) - { - Byte buf[LZMA_PROPS_SIZE]; - - if (Slave_->Load(buf, sizeof(buf)) != sizeof(buf)) - ythrow yexception() << "can't read lzma header"; - - Check(LzmaDec_Allocate(&H_, buf, sizeof(buf), Alloc())); - LzmaDec_Init(&H_); - } - -private: - bool Fill() override { - size_t size = Slave_->Read(In_, sizeof(In_)); - InBegin_ = In_; - InEnd_ = In_ + size; - - return size; - } - -private: + : Slave_(slave) + { + Byte buf[LZMA_PROPS_SIZE]; + + if (Slave_->Load(buf, sizeof(buf)) != sizeof(buf)) + ythrow yexception() << "can't read lzma header"; + + Check(LzmaDec_Allocate(&H_, buf, sizeof(buf), Alloc())); + LzmaDec_Init(&H_); + } + +private: + bool Fill() override { + size_t size = Slave_->Read(In_, sizeof(In_)); + InBegin_ = In_; + InEnd_ = In_ + size; + + return size; + } + +private: IInputStream* Slave_; - char In_[4096]; -}; - + char In_[4096]; +}; + class TLzmaDecompress::TImplZeroCopy: public TLzmaDecompress::TImpl { public: inline TImplZeroCopy(IZeroCopyInput* in) : Input_(in) - { - if (!Fill()) - ythrow yexception() << "can't read lzma header"; - - char buf[LZMA_PROPS_SIZE]; - char* header; - if (InEnd_ - InBegin_ >= LZMA_PROPS_SIZE) { - header = InBegin_; - InBegin_ += LZMA_PROPS_SIZE; - } else { - //bad luck, first part is less than header - //try to copy header part by part to the local buffer - const char* end = buf + sizeof(buf); - char* pos = buf; - while (1) { - size_t left = end - pos; - size_t avail = InEnd_ - InBegin_; - if (left < avail) { - memcpy(pos, InBegin_, left); - InBegin_ += left; - break; - } else { - memcpy(pos, InBegin_, avail); - pos += avail; - if (!Fill()) { - ythrow yexception() << "can't read lzma header"; + { + if (!Fill()) + ythrow yexception() << "can't read lzma header"; + + char buf[LZMA_PROPS_SIZE]; + char* header; + if (InEnd_ - InBegin_ >= LZMA_PROPS_SIZE) { + header = InBegin_; + InBegin_ += LZMA_PROPS_SIZE; + } else { + //bad luck, first part is less than header + //try to copy header part by part to the local buffer + const char* end = buf + sizeof(buf); + char* pos = buf; + while (1) { + size_t left = end - pos; + size_t avail = InEnd_ - InBegin_; + if (left < avail) { + memcpy(pos, InBegin_, left); + InBegin_ += left; + break; + } else { + memcpy(pos, InBegin_, avail); + pos += avail; + if (!Fill()) { + ythrow yexception() << "can't read lzma header"; } } } - header = buf; - } + header = buf; + } - Check(LzmaDec_Allocate(&H_, (Byte*)header, LZMA_PROPS_SIZE, Alloc())); + Check(LzmaDec_Allocate(&H_, (Byte*)header, LZMA_PROPS_SIZE, Alloc())); - LzmaDec_Init(&H_); - } + LzmaDec_Init(&H_); + } private: - bool Fill() override { + bool Fill() override { size_t size = Input_->Next(&InBegin_); - + if (size) { - InEnd_ = InBegin_ + size; - + InEnd_ = InBegin_ + size; + return true; - } - - return false; - } + } + + return false; + } IZeroCopyInput* Input_; }; TLzmaCompress::TLzmaCompress(IOutputStream* slave, size_t level) - : Impl_(new TImpl(slave, level)) -{ -} - + : Impl_(new TImpl(slave, level)) +{ +} + TLzmaCompress::~TLzmaCompress() { -} - -void TLzmaCompress::DoWrite(const void* buf, size_t len) { - if (!Impl_) { - ythrow yexception() << "can not write to finished lzma stream"; - } - - Impl_->Write(buf, len); -} - -void TLzmaCompress::DoFinish() { +} + +void TLzmaCompress::DoWrite(const void* buf, size_t len) { + if (!Impl_) { + ythrow yexception() << "can not write to finished lzma stream"; + } + + Impl_->Write(buf, len); +} + +void TLzmaCompress::DoFinish() { THolder<TImpl> impl(Impl_.Release()); if (impl) { impl->Finish(); - } -} - + } +} + TLzmaDecompress::TLzmaDecompress(IInputStream* slave) : Impl_(new TImplStream(slave)) -{ -} - +{ +} + TLzmaDecompress::TLzmaDecompress(IZeroCopyInput* input) : Impl_(new TImplZeroCopy(input)) { } TLzmaDecompress::~TLzmaDecompress() { -} - -size_t TLzmaDecompress::DoRead(void* buf, size_t len) { - return Impl_->Read(buf, len); -} +} + +size_t TLzmaDecompress::DoRead(void* buf, size_t len) { + return Impl_->Read(buf, len); +} diff --git a/library/cpp/streams/lzma/lzma.h b/library/cpp/streams/lzma/lzma.h index ca1e06e9ef..af5399c73c 100644 --- a/library/cpp/streams/lzma/lzma.h +++ b/library/cpp/streams/lzma/lzma.h @@ -1,37 +1,37 @@ #pragma once - -#include <util/stream/input.h> -#include <util/stream/output.h> -#include <util/stream/zerocopy.h> - -#include <util/generic/ptr.h> - + +#include <util/stream/input.h> +#include <util/stream/output.h> +#include <util/stream/zerocopy.h> + +#include <util/generic/ptr.h> + class TLzmaCompress: public IOutputStream { -public: +public: TLzmaCompress(IOutputStream* slave, size_t level = 7); ~TLzmaCompress() override; - -private: + +private: void DoWrite(const void* buf, size_t len) override; void DoFinish() override; - -private: - class TImpl; - THolder<TImpl> Impl_; -}; - + +private: + class TImpl; + THolder<TImpl> Impl_; +}; + class TLzmaDecompress: public IInputStream { -public: +public: TLzmaDecompress(IInputStream* slave); TLzmaDecompress(IZeroCopyInput* input); ~TLzmaDecompress() override; - -private: + +private: size_t DoRead(void* buf, size_t len) override; - -private: - class TImpl; - class TImplStream; - class TImplZeroCopy; - THolder<TImpl> Impl_; -}; + +private: + class TImpl; + class TImplStream; + class TImplZeroCopy; + THolder<TImpl> Impl_; +}; diff --git a/library/cpp/streams/lzma/lzma_ut.cpp b/library/cpp/streams/lzma/lzma_ut.cpp index 847e98d1ca..47a3aa4bf3 100644 --- a/library/cpp/streams/lzma/lzma_ut.cpp +++ b/library/cpp/streams/lzma/lzma_ut.cpp @@ -1,127 +1,127 @@ -#include "lzma.h" - +#include "lzma.h" + #include <library/cpp/testing/unittest/registar.h> - -#include <util/stream/mem.h> -#include <util/random/fast.h> -#include <util/random/random.h> - + +#include <util/stream/mem.h> +#include <util/random/fast.h> +#include <util/random/random.h> + class TStrokaByOneByte: public IZeroCopyInput { -public: +public: TStrokaByOneByte(const TString& s) - : Data(s) - , Pos(s.data()) - { - } + : Data(s) + , Pos(s.data()) + { + } -private: +private: size_t DoNext(const void** ptr, size_t len) override { - if (Pos < Data.end()) { + if (Pos < Data.end()) { len = Min(len, static_cast<size_t>(1)); *ptr = Pos; Pos += len; return len; - } else { + } else { return 0; } - } + } TString Data; - const char* Pos; + const char* Pos; }; -class TLzmaTest: public TTestBase { - UNIT_TEST_SUITE(TLzmaTest); - UNIT_TEST(Test1) - UNIT_TEST(Test2) - UNIT_TEST_SUITE_END(); - -private: +class TLzmaTest: public TTestBase { + UNIT_TEST_SUITE(TLzmaTest); + UNIT_TEST(Test1) + UNIT_TEST(Test2) + UNIT_TEST_SUITE_END(); + +private: inline TString GenData() { TString data; - TReallyFastRng32 rnd(RandomNumber<ui64>()); - - for (size_t i = 0; i < 50000; ++i) { - const char ch = rnd.Uniform(256); - const size_t len = 1 + rnd.Uniform(10); - + TReallyFastRng32 rnd(RandomNumber<ui64>()); + + for (size_t i = 0; i < 50000; ++i) { + const char ch = rnd.Uniform(256); + const size_t len = 1 + rnd.Uniform(10); + data += TString(len, ch); - } - - return data; - } - - inline void Test2() { + } + + return data; + } + + inline void Test2() { class TExcOutput: public IOutputStream { - public: + public: ~TExcOutput() override { - } - + } + void DoWrite(const void*, size_t) override { - throw 12345; - } - }; - + throw 12345; + } + }; + TString data(GenData()); TMemoryInput mi(data.data(), data.size()); - TExcOutput out; - - try { - TLzmaCompress c(&out); - + TExcOutput out; + + try { + TLzmaCompress c(&out); + TransferData(&mi, &c); - } catch (int i) { - UNIT_ASSERT_EQUAL(i, 12345); - } - } - - inline void Test1() { + } catch (int i) { + UNIT_ASSERT_EQUAL(i, 12345); + } + } + + inline void Test1() { TString data(GenData()); TString data1; TString res; - - { + + { TMemoryInput mi(data.data(), data.size()); - TStringOutput so(res); - TLzmaCompress c(&so); - + TStringOutput so(res); + TLzmaCompress c(&so); + TransferData(&mi, &c); - - c.Finish(); - } - - { + + c.Finish(); + } + + { TMemoryInput mi(res.data(), res.size()); - TStringOutput so(data1); + TStringOutput so(data1); TLzmaDecompress d((IInputStream*)&mi); - - TransferData(&d, &so); - } - - UNIT_ASSERT_EQUAL(data, data1); - - data1.clear(); - { + + TransferData(&d, &so); + } + + UNIT_ASSERT_EQUAL(data, data1); + + data1.clear(); + { TMemoryInput mi(res.data(), res.size()); - TStringOutput so(data1); + TStringOutput so(data1); TLzmaDecompress d(&mi); - TransferData(&d, &so); - } + TransferData(&d, &so); + } - UNIT_ASSERT_EQUAL(data, data1); + UNIT_ASSERT_EQUAL(data, data1); - data1.clear(); - { - TStrokaByOneByte mi(res); - TStringOutput so(data1); + data1.clear(); + { + TStrokaByOneByte mi(res); + TStringOutput so(data1); TLzmaDecompress d(&mi); - TransferData(&d, &so); - } - - UNIT_ASSERT_EQUAL(data, data1); - } -}; + TransferData(&d, &so); + } -UNIT_TEST_SUITE_REGISTRATION(TLzmaTest); + UNIT_ASSERT_EQUAL(data, data1); + } +}; + +UNIT_TEST_SUITE_REGISTRATION(TLzmaTest); diff --git a/library/cpp/streams/lzma/ut/ya.make b/library/cpp/streams/lzma/ut/ya.make index 01624f0259..81981221a7 100644 --- a/library/cpp/streams/lzma/ut/ya.make +++ b/library/cpp/streams/lzma/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/streams/lzma) - -OWNER( - pg + +OWNER( + pg g:util -) - -SRCS( - lzma_ut.cpp -) - -END() +) + +SRCS( + lzma_ut.cpp +) + +END() diff --git a/library/cpp/streams/lzma/ya.make b/library/cpp/streams/lzma/ya.make index 38c05145c4..dc96e8b7fe 100644 --- a/library/cpp/streams/lzma/ya.make +++ b/library/cpp/streams/lzma/ya.make @@ -1,16 +1,16 @@ -LIBRARY() - -OWNER( - pg +LIBRARY() + +OWNER( + pg g:util -) - -PEERDIR( - contrib/libs/lzmasdk -) - -SRCS( - lzma.cpp -) - -END() +) + +PEERDIR( + contrib/libs/lzmasdk +) + +SRCS( + lzma.cpp +) + +END() diff --git a/library/cpp/streams/ya.make b/library/cpp/streams/ya.make index 7426a874ee..67ffb63236 100644 --- a/library/cpp/streams/ya.make +++ b/library/cpp/streams/ya.make @@ -1,26 +1,26 @@ -RECURSE( +RECURSE( + brotli + brotli/ut + base64 brotli - brotli/ut - base64 - brotli - bzip2 - bzip2/ut - factory - factory/ut - fields_io - fields_io/ut + bzip2 + bzip2/ut + factory + factory/ut + fields_io + fields_io/ut growing_file_input growing_file_input/ut - lz - lz/ut - lzma - lzma/ut + lz + lz/ut + lzma + lzma/ut lzop lzop/ut - special - special/ut + special + special/ut xz - zc_memory_input + zc_memory_input zstd zstd/ut -) +) diff --git a/library/cpp/streams/zc_memory_input/ya.make b/library/cpp/streams/zc_memory_input/ya.make index bc94d6f1ed..c86449b1b5 100644 --- a/library/cpp/streams/zc_memory_input/ya.make +++ b/library/cpp/streams/zc_memory_input/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - +LIBRARY() + OWNER( pg g:util ) - -SRCS( - zc_memory_input.cpp -) - -END() + +SRCS( + zc_memory_input.cpp +) + +END() diff --git a/library/cpp/streams/zc_memory_input/zc_memory_input.cpp b/library/cpp/streams/zc_memory_input/zc_memory_input.cpp index 682099a239..398344153e 100644 --- a/library/cpp/streams/zc_memory_input/zc_memory_input.cpp +++ b/library/cpp/streams/zc_memory_input/zc_memory_input.cpp @@ -1 +1 @@ -#include "zc_memory_input.h" +#include "zc_memory_input.h" diff --git a/library/cpp/streams/zc_memory_input/zc_memory_input.h b/library/cpp/streams/zc_memory_input/zc_memory_input.h index c939d8e426..2ffda1504e 100644 --- a/library/cpp/streams/zc_memory_input/zc_memory_input.h +++ b/library/cpp/streams/zc_memory_input/zc_memory_input.h @@ -2,23 +2,23 @@ #include <util/stream/mem.h> #include <util/system/defaults.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> /// Zero-copy memory input with fixed read -class TZCMemoryInput: public TMemoryInput { +class TZCMemoryInput: public TMemoryInput { public: - TZCMemoryInput() { - } + TZCMemoryInput() { + } TZCMemoryInput(const char* dataPtr, size_t size) : TMemoryInput(dataPtr, size) - { - } + { + } TZCMemoryInput(TMemoryInput& rhs) : TMemoryInput(rhs.Buf(), rhs.Avail()) - { - } + { + } /// if there's 'size' data read it, otherwise just return false Y_FORCE_INLINE bool ReadFixed(const char*& buf, size_t size) { @@ -30,7 +30,7 @@ public: return false; } - template <class T> + template <class T> Y_FORCE_INLINE T LoadPOD() { const char* buf = nullptr; if (!ReadFixed(buf, sizeof(T))) @@ -41,7 +41,7 @@ public: return res; } - template <class T> + template <class T> Y_FORCE_INLINE void ReadPOD(T& x) { x = LoadPOD<T>(); } diff --git a/library/cpp/string_utils/base64/base64.cpp b/library/cpp/string_utils/base64/base64.cpp index 05c201f0de..08533d2b47 100644 --- a/library/cpp/string_utils/base64/base64.cpp +++ b/library/cpp/string_utils/base64/base64.cpp @@ -1,5 +1,5 @@ -#include "base64.h" - +#include "base64.h" + #include <contrib/libs/base64/avx2/libbase64.h> #include <contrib/libs/base64/ssse3/libbase64.h> #include <contrib/libs/base64/neon32/libbase64.h> @@ -7,10 +7,10 @@ #include <contrib/libs/base64/plain32/libbase64.h> #include <contrib/libs/base64/plain64/libbase64.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <util/system/cpu_id.h> #include <util/system/platform.h> - + #include <cstdlib> namespace { @@ -71,7 +71,7 @@ namespace { } } -static const char base64_etab_std[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +static const char base64_etab_std[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; static const char base64_bkw[] = { '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', // 0..15 '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', // 16..31 @@ -93,7 +93,7 @@ static const char base64_bkw[] = { static_assert(Y_ARRAY_SIZE(base64_bkw) == 256, "wrong size"); // Base64 for url encoding, RFC3548 -static const char base64_etab_url[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; +static const char base64_etab_url[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; static inline unsigned char GetBase64EncodedIndex0(unsigned char octet0) { return (octet0 >> 2); @@ -111,13 +111,13 @@ static inline unsigned char GetBase64EncodedIndex3(unsigned char octet2) { return (octet2 & 0x3f); } -template <bool urlVersion> -static inline char* Base64EncodeImpl(char* outstr, const unsigned char* instr, size_t len) { +template <bool urlVersion> +static inline char* Base64EncodeImpl(char* outstr, const unsigned char* instr, size_t len) { const char* const base64_etab = (urlVersion ? base64_etab_url : base64_etab_std); const char pad = (urlVersion ? ',' : '='); - size_t idx = 0; - + size_t idx = 0; + while (idx + 2 < len) { *outstr++ = base64_etab[GetBase64EncodedIndex0(instr[idx])]; *outstr++ = base64_etab[GetBase64EncodedIndex1(instr[idx], instr[idx + 1])]; @@ -136,44 +136,44 @@ static inline char* Base64EncodeImpl(char* outstr, const unsigned char* instr, s } *outstr++ = pad; } - *outstr = 0; - + *outstr = 0; + return outstr; } static char* Base64EncodePlain(char* outstr, const unsigned char* instr, size_t len) { - return Base64EncodeImpl<false>(outstr, instr, len); + return Base64EncodeImpl<false>(outstr, instr, len); } -char* Base64EncodeUrl(char* outstr, const unsigned char* instr, size_t len) { - return Base64EncodeImpl<true>(outstr, instr, len); +char* Base64EncodeUrl(char* outstr, const unsigned char* instr, size_t len) { + return Base64EncodeImpl<true>(outstr, instr, len); } -inline void uudecode_1(char* dst, unsigned char* src) { +inline void uudecode_1(char* dst, unsigned char* src) { dst[0] = char((base64_bkw[src[0]] << 2) | (base64_bkw[src[1]] >> 4)); dst[1] = char((base64_bkw[src[1]] << 4) | (base64_bkw[src[2]] >> 2)); dst[2] = char((base64_bkw[src[2]] << 6) | base64_bkw[src[3]]); } static size_t Base64DecodePlain(void* dst, const char* b, const char* e) { - size_t n = 0; - while (b < e) { - uudecode_1((char*)dst + n, (unsigned char*)b); - - b += 4; - n += 3; - } - + size_t n = 0; + while (b < e) { + uudecode_1((char*)dst + n, (unsigned char*)b); + + b += 4; + n += 3; + } + if (n > 0) { - if (b[-1] == ',' || b[-1] == '=') { + if (b[-1] == ',' || b[-1] == '=') { n--; - - if (b[-2] == ',' || b[-2] == '=') { + + if (b[-2] == ',' || b[-2] == '=') { n--; - } + } } } - + return n; } @@ -189,9 +189,9 @@ static const char base64_bkw_strict[] = "\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100\100"; size_t Base64StrictDecode(void* out, const char* b, const char* e) { - char* dst = (char*)out; - const unsigned char* src = (unsigned char*)b; - const unsigned char* const end = (unsigned char*)e; + char* dst = (char*)out; + const unsigned char* src = (unsigned char*)b; + const unsigned char* const end = (unsigned char*)e; Y_ENSURE(!((e - b) % 4), "incorrect input length for base64 decode"); @@ -228,7 +228,7 @@ size_t Base64StrictDecode(void* out, const char* b, const char* e) { } } - return dst - (char*)out; + return dst - (char*)out; } size_t Base64Decode(void* dst, const char* b, const char* e) { diff --git a/library/cpp/string_utils/base64/base64.h b/library/cpp/string_utils/base64/base64.h index f778a6425a..2f59572c51 100644 --- a/library/cpp/string_utils/base64/base64.h +++ b/library/cpp/string_utils/base64/base64.h @@ -1,15 +1,15 @@ #pragma once - + #include <util/system/defaults.h> -#include <util/generic/strbuf.h> +#include <util/generic/strbuf.h> #include <util/generic/string.h> - + /* @return Size of the buffer required to decode Base64 encoded data of size `len`. */ constexpr size_t Base64DecodeBufSize(const size_t len) noexcept { return (len + 3) / 4 * 3; -} - +} + /* Decode Base64 encoded data. Can decode both regular Base64 and Base64URL encoded data. Can decode * only valid Base64[URL] data, behaviour for invalid data is unspecified. * @@ -21,12 +21,12 @@ constexpr size_t Base64DecodeBufSize(const size_t len) noexcept { * * @return Return number of bytes decoded. */ -size_t Base64Decode(void* dst, const char* b, const char* e); - +size_t Base64Decode(void* dst, const char* b, const char* e); + inline TStringBuf Base64Decode(const TStringBuf src, void* dst) { - return TStringBuf((const char*)dst, Base64Decode(dst, src.begin(), src.end())); -} - + return TStringBuf((const char*)dst, Base64Decode(dst, src.begin(), src.end())); +} + inline void Base64Decode(const TStringBuf src, TString& dst) { dst.ReserveAndResize(Base64DecodeBufSize(src.size())); dst.resize(Base64Decode(src, dst.begin()).size()); @@ -36,9 +36,9 @@ inline void Base64Decode(const TStringBuf src, TString& dst) { inline TString Base64Decode(const TStringBuf s) { TString ret; Base64Decode(s, ret); - return ret; -} - + return ret; +} + /// /// @brief Decodes Base64 string with strict verification /// of invalid symbols, also tries to decode Base64 string with padding @@ -64,7 +64,7 @@ size_t Base64StrictDecode(void* dst, const char* b, const char* e); /// @return Returns dst wrapped into TStringBuf. /// inline TStringBuf Base64StrictDecode(const TStringBuf src, void* dst) { - return TStringBuf((const char*)dst, Base64StrictDecode(dst, src.begin(), src.end())); + return TStringBuf((const char*)dst, Base64StrictDecode(dst, src.begin(), src.end())); } /// @@ -91,22 +91,22 @@ inline TString Base64StrictDecode(const TStringBuf src) { /// Works with strings which length is not divisible by 4. TString Base64DecodeUneven(const TStringBuf s); -//encode +//encode constexpr size_t Base64EncodeBufSize(const size_t len) noexcept { return (len + 2) / 3 * 4 + 1; -} - -char* Base64Encode(char* outstr, const unsigned char* instr, size_t len); -char* Base64EncodeUrl(char* outstr, const unsigned char* instr, size_t len); - +} + +char* Base64Encode(char* outstr, const unsigned char* instr, size_t len); +char* Base64EncodeUrl(char* outstr, const unsigned char* instr, size_t len); + inline TStringBuf Base64Encode(const TStringBuf src, void* tmp) { return TStringBuf((const char*)tmp, Base64Encode((char*)tmp, (const unsigned char*)src.data(), src.size())); -} - +} + inline TStringBuf Base64EncodeUrl(const TStringBuf src, void* tmp) { return TStringBuf((const char*)tmp, Base64EncodeUrl((char*)tmp, (const unsigned char*)src.data(), src.size())); -} - +} + inline void Base64Encode(const TStringBuf src, TString& dst) { dst.ReserveAndResize(Base64EncodeBufSize(src.size())); dst.resize(Base64Encode(src, dst.begin()).size()); @@ -120,11 +120,11 @@ inline void Base64EncodeUrl(const TStringBuf src, TString& dst) { inline TString Base64Encode(const TStringBuf s) { TString ret; Base64Encode(s, ret); - return ret; -} - + return ret; +} + inline TString Base64EncodeUrl(const TStringBuf s) { TString ret; Base64EncodeUrl(s, ret); - return ret; -} + return ret; +} diff --git a/library/cpp/string_utils/base64/base64_ut.cpp b/library/cpp/string_utils/base64/base64_ut.cpp index bcc1e65879..16eb9c1bf2 100644 --- a/library/cpp/string_utils/base64/base64_ut.cpp +++ b/library/cpp/string_utils/base64/base64_ut.cpp @@ -1,5 +1,5 @@ -#include "base64.h" - +#include "base64.h" + #include <contrib/libs/base64/avx2/libbase64.h> #include <contrib/libs/base64/neon32/libbase64.h> #include <contrib/libs/base64/neon64/libbase64.h> @@ -197,11 +197,11 @@ static void TestEncodeStrictDecodeIntoString(const TString& plain, const TString Y_UNIT_TEST_SUITE(TBase64) { Y_UNIT_TEST(TestEncode) { - UNIT_ASSERT_VALUES_EQUAL(Base64Encode("12z"), "MTJ6"); - UNIT_ASSERT_VALUES_EQUAL(Base64Encode("123"), "MTIz"); - UNIT_ASSERT_VALUES_EQUAL(Base64Encode("12"), "MTI="); - UNIT_ASSERT_VALUES_EQUAL(Base64Encode("1"), "MQ=="); - } + UNIT_ASSERT_VALUES_EQUAL(Base64Encode("12z"), "MTJ6"); + UNIT_ASSERT_VALUES_EQUAL(Base64Encode("123"), "MTIz"); + UNIT_ASSERT_VALUES_EQUAL(Base64Encode("12"), "MTI="); + UNIT_ASSERT_VALUES_EQUAL(Base64Encode("1"), "MQ=="); + } Y_UNIT_TEST(TestIntoString) { { @@ -248,7 +248,7 @@ Y_UNIT_TEST_SUITE(TBase64) { UNIT_ASSERT_VALUES_EQUAL(Base64Decode(""), ""); UNIT_ASSERT_VALUES_EQUAL(Base64StrictDecode(""), ""); - UNIT_ASSERT_VALUES_EQUAL(Base64Decode("MTI="), "12"); + UNIT_ASSERT_VALUES_EQUAL(Base64Decode("MTI="), "12"); UNIT_ASSERT_VALUES_EQUAL(Base64StrictDecode("MTI="), "12"); UNIT_ASSERT_VALUES_EQUAL(Base64Decode("QQ=="), "A"); @@ -265,7 +265,7 @@ Y_UNIT_TEST_SUITE(TBase64) { UNIT_ASSERT_EXCEPTION(Base64StrictDecode("aHh=="), yexception); UNIT_ASSERT_EXCEPTION(Base64StrictDecode("\1\1\1\2"), yexception); - } + } Y_UNIT_TEST(TestDecodeUneven) { UNIT_ASSERT_VALUES_EQUAL(Base64DecodeUneven(""), ""); @@ -494,4 +494,4 @@ Y_UNIT_TEST_SUITE(TBase64) { const auto xDec = Base64Decode("SG93IGRvIEkgY29udmVydCBiZXR3ZWVuIGJpZy1lbmRpYW4gYW5kIGxpdHRsZS1lbmRpYW4gdmFsdWVzIGluIEMrKz9hYQ,,"); UNIT_ASSERT_VALUES_EQUAL(x, xDec); } -} +} diff --git a/library/cpp/string_utils/indent_text/indent_text.cpp b/library/cpp/string_utils/indent_text/indent_text.cpp index 09a4f6bca8..e2fb1d69d0 100644 --- a/library/cpp/string_utils/indent_text/indent_text.cpp +++ b/library/cpp/string_utils/indent_text/indent_text.cpp @@ -1,5 +1,5 @@ -#include "indent_text.h" - +#include "indent_text.h" + #include <util/stream/str.h> TString IndentText(TStringBuf text, TStringBuf indent) { diff --git a/library/cpp/string_utils/indent_text/ya.make b/library/cpp/string_utils/indent_text/ya.make index cd0ed9ec61..9757fde208 100644 --- a/library/cpp/string_utils/indent_text/ya.make +++ b/library/cpp/string_utils/indent_text/ya.make @@ -1,9 +1,9 @@ -LIBRARY() - -OWNER(nga) - -SRCS( - indent_text.cpp -) - -END() +LIBRARY() + +OWNER(nga) + +SRCS( + indent_text.cpp +) + +END() diff --git a/library/cpp/string_utils/levenshtein_diff/levenshtein_diff.h b/library/cpp/string_utils/levenshtein_diff/levenshtein_diff.h index 8a240bfed8..e3d3f5247e 100644 --- a/library/cpp/string_utils/levenshtein_diff/levenshtein_diff.h +++ b/library/cpp/string_utils/levenshtein_diff/levenshtein_diff.h @@ -9,34 +9,34 @@ #include <utility> namespace NLevenshtein { - enum EEditMoveType { - EMT_SPECIAL, - EMT_PRESERVE, - EMT_REPLACE, - EMT_DELETE, - EMT_INSERT - }; - - inline bool IsImportantEditMove(EEditMoveType p) { - return (p != EMT_SPECIAL && p != EMT_PRESERVE); - } - - inline void MakeMove(EEditMoveType t, int& p1, int& p2) { - switch (t) { - case EMT_PRESERVE: - case EMT_REPLACE: - p1++; - p2++; - break; - case EMT_DELETE: - p1++; - break; - case EMT_INSERT: - p2++; - break; - default: - break; - } + enum EEditMoveType { + EMT_SPECIAL, + EMT_PRESERVE, + EMT_REPLACE, + EMT_DELETE, + EMT_INSERT + }; + + inline bool IsImportantEditMove(EEditMoveType p) { + return (p != EMT_SPECIAL && p != EMT_PRESERVE); + } + + inline void MakeMove(EEditMoveType t, int& p1, int& p2) { + switch (t) { + case EMT_PRESERVE: + case EMT_REPLACE: + p1++; + p2++; + break; + case EMT_DELETE: + p1++; + break; + case EMT_INSERT: + p2++; + break; + default: + break; + } } using TEditChain = TVector<EEditMoveType>; @@ -58,7 +58,7 @@ namespace NLevenshtein { template <typename TStringType> using TCharType = typename std::decay_t<decltype(std::add_const_t<TStringType>()[0])>; - /// Finds sequence of "edit moves" for two strings + /// Finds sequence of "edit moves" for two strings template <class TStringType, class TWeightType = int, class TReplaceWeigher = TWeightOneBinaryGetter<TCharType<TStringType>>, class TDeleteWeigher = TWeightOneUnaryGetter<TCharType<TStringType>>, @@ -69,20 +69,20 @@ namespace NLevenshtein { const TDeleteWeigher& deleteWeigher = TDeleteWeigher(), const TInsertWeigher& insertWeigher = TInsertWeigher()) { - int l1 = (int)str1.size(); - int l2 = (int)str2.size(); + int l1 = (int)str1.size(); + int l2 = (int)str2.size(); TMatrix<std::pair<TWeightType, EEditMoveType>> ma(l1 + 1, l2 + 1); /// ma[i][j].first = diff(str1[0..i-1], str2[0..j-1]) - ma[0][0] = std::make_pair(0, EMT_SPECIAL); // starting point - for (int i = 1; i <= l1; i++) { + ma[0][0] = std::make_pair(0, EMT_SPECIAL); // starting point + for (int i = 1; i <= l1; i++) { ma[i][0] = std::make_pair(ma[i - 1][0].first + deleteWeigher(str1[i - 1]), EMT_DELETE); } - for (int i = 1; i <= l2; i++) { + for (int i = 1; i <= l2; i++) { ma[0][i] = std::make_pair(ma[0][i - 1].first + insertWeigher(str2[i - 1]), EMT_INSERT); } - // Here goes basic Levestein's algorithm - for (int i = 1; i <= l1; i++) { - for (int j = 1; j <= l2; j++) { + // Here goes basic Levestein's algorithm + for (int i = 1; i <= l1; i++) { + for (int j = 1; j <= l2; j++) { if (str1[i - 1] == str2[j - 1]) { ma[i][j] = std::make_pair(ma[i - 1][j - 1].first, EMT_PRESERVE); } else { @@ -108,31 +108,31 @@ namespace NLevenshtein { ma[i][j] = std::make_pair(insertPathWeight, EMT_INSERT); } } - } - } - // Tracing the path from final point - res.clear(); + } + } + // Tracing the path from final point + res.clear(); res.reserve(Max<size_t>(l1, l2)); - for (int i = l1, j = l2; ma[i][j].second != EMT_SPECIAL;) { - res.push_back(ma[i][j].second); - switch (ma[i][j].second) { - case EMT_PRESERVE: - case EMT_REPLACE: - --i; - --j; - break; - case EMT_DELETE: - --i; - break; - case EMT_INSERT: - --j; - break; - default: - // TODO: throw exception - break; - } - } - std::reverse(res.begin(), res.end()); + for (int i = l1, j = l2; ma[i][j].second != EMT_SPECIAL;) { + res.push_back(ma[i][j].second); + switch (ma[i][j].second) { + case EMT_PRESERVE: + case EMT_REPLACE: + --i; + --j; + break; + case EMT_DELETE: + --i; + break; + case EMT_INSERT: + --j; + break; + default: + // TODO: throw exception + break; + } + } + std::reverse(res.begin(), res.end()); if (weight != nullptr) { *weight = ma[l1][l2].first; @@ -151,9 +151,9 @@ namespace NLevenshtein { return result; } - /// Calculates substrings to be replaced for str1->str2 transformation - struct TReplacement { - int CorrectOffset, CorrectLength, MisspelledOffset, MisspelledLength; + /// Calculates substrings to be replaced for str1->str2 transformation + struct TReplacement { + int CorrectOffset, CorrectLength, MisspelledOffset, MisspelledLength; TReplacement() : CorrectOffset(0) , CorrectLength(0) @@ -161,32 +161,32 @@ namespace NLevenshtein { , MisspelledLength(0) { } - TReplacement(int correctOffset, int correctLength, int misspelledOffset, int misspelledLength) - : CorrectOffset(correctOffset) - , CorrectLength(correctLength) - , MisspelledOffset(misspelledOffset) - , MisspelledLength(misspelledLength) - { - } - }; + TReplacement(int correctOffset, int correctLength, int misspelledOffset, int misspelledLength) + : CorrectOffset(correctOffset) + , CorrectLength(correctLength) + , MisspelledOffset(misspelledOffset) + , MisspelledLength(misspelledLength) + { + } + }; template <class TStringType> void GetStringReplacements(const TStringType& str1, const TStringType& str2, TVector<TReplacement>& res) { - TEditChain editChain; - GetEditChain(str1, str2, editChain); - editChain.push_back(EMT_SPECIAL); - int c1 = 0, c2 = 0; - res.clear(); - for (TEditChain::const_iterator it = editChain.begin(); it != editChain.end(); it++) { - if (IsImportantEditMove(*it)) { - int sc1 = c1, sc2 = c2; - do { - MakeMove(*it, c1, c2); - ++it; - } while (IsImportantEditMove(*it)); - res.push_back(TReplacement(sc1, c1 - sc1, sc2, c2 - sc2)); - } - MakeMove(*it, c1, c2); - } + TEditChain editChain; + GetEditChain(str1, str2, editChain); + editChain.push_back(EMT_SPECIAL); + int c1 = 0, c2 = 0; + res.clear(); + for (TEditChain::const_iterator it = editChain.begin(); it != editChain.end(); it++) { + if (IsImportantEditMove(*it)) { + int sc1 = c1, sc2 = c2; + do { + MakeMove(*it, c1, c2); + ++it; + } while (IsImportantEditMove(*it)); + res.push_back(TReplacement(sc1, c1 - sc1, sc2, c2 - sc2)); + } + MakeMove(*it, c1, c2); + } } } diff --git a/library/cpp/string_utils/levenshtein_diff/ya.make b/library/cpp/string_utils/levenshtein_diff/ya.make index bafefe5365..e51c08aa1e 100644 --- a/library/cpp/string_utils/levenshtein_diff/ya.make +++ b/library/cpp/string_utils/levenshtein_diff/ya.make @@ -1,13 +1,13 @@ -LIBRARY() - +LIBRARY() + OWNER(g:mt) - -SRCS( + +SRCS( levenshtein_diff.cpp -) - -PEERDIR( - util/draft -) - -END() +) + +PEERDIR( + util/draft +) + +END() diff --git a/library/cpp/string_utils/parse_size/parse_size.cpp b/library/cpp/string_utils/parse_size/parse_size.cpp index 39188d560b..1195f89a51 100644 --- a/library/cpp/string_utils/parse_size/parse_size.cpp +++ b/library/cpp/string_utils/parse_size/parse_size.cpp @@ -2,89 +2,89 @@ #include <util/generic/yexception.h> #include <util/generic/ylimits.h> -#include <util/string/cast.h> +#include <util/string/cast.h> #include <util/stream/output.h> namespace { - enum ESuffixShifts { - ESS_KILO_BYTES = 10, - ESS_MEGA_BYTES = 20, - ESS_GIGA_BYTES = 30, - ESS_TERA_BYTES = 40, - }; - - bool TryShiftValue(ui64& value, ui64 shift) { - if (value > (Max<ui64>() >> shift)) { - return false; - } - - value <<= shift; - return true; + enum ESuffixShifts { + ESS_KILO_BYTES = 10, + ESS_MEGA_BYTES = 20, + ESS_GIGA_BYTES = 30, + ESS_TERA_BYTES = 40, + }; + + bool TryShiftValue(ui64& value, ui64 shift) { + if (value > (Max<ui64>() >> shift)) { + return false; + } + + value <<= shift; + return true; } - ui64 ShiftValue(ui64 value, ui64 shift) { - if (!TryShiftValue(value, shift)) { - ythrow yexception() << "value overflow '" << value << " << " << shift << "'"; - } else { - return value; - } + ui64 ShiftValue(ui64 value, ui64 shift) { + if (!TryShiftValue(value, shift)) { + ythrow yexception() << "value overflow '" << value << " << " << shift << "'"; + } else { + return value; + } } } namespace NSize { - ui64 ParseSize(TStringBuf str) { + ui64 ParseSize(TStringBuf str) { if (! str.size()) - ythrow yexception() << "Wrong size " << str; + ythrow yexception() << "Wrong size " << str; char suff = tolower(str[str.size() - 1]); - if (isdigit(suff)) - return FromString<ui64>(str); - ui64 shift = 1; - switch (suff) { - case 'k': - shift = ESS_KILO_BYTES; - break; - case 'm': - shift = ESS_MEGA_BYTES; - break; - case 'g': - shift = ESS_GIGA_BYTES; - break; - case 't': - shift = ESS_TERA_BYTES; - break; - default: - ythrow yexception() << "Unknown suffix " << str; - } + if (isdigit(suff)) + return FromString<ui64>(str); + ui64 shift = 1; + switch (suff) { + case 'k': + shift = ESS_KILO_BYTES; + break; + case 'm': + shift = ESS_MEGA_BYTES; + break; + case 'g': + shift = ESS_GIGA_BYTES; + break; + case 't': + shift = ESS_TERA_BYTES; + break; + default: + ythrow yexception() << "Unknown suffix " << str; + } ui64 value = FromString<ui64>(str.substr(0, str.size() - 1)); - - if (!TryShiftValue(value, shift)) { - ythrow yexception() << "Value overflow " << str; - } else { - return value; - } + + if (!TryShiftValue(value, shift)) { + ythrow yexception() << "Value overflow " << str; + } else { + return value; + } } - TSize FromKiloBytes(ui64 value) { - return TSize(ShiftValue(value, ESS_KILO_BYTES)); - } + TSize FromKiloBytes(ui64 value) { + return TSize(ShiftValue(value, ESS_KILO_BYTES)); + } - TSize FromMegaBytes(ui64 value) { - return TSize(ShiftValue(value, ESS_MEGA_BYTES)); + TSize FromMegaBytes(ui64 value) { + return TSize(ShiftValue(value, ESS_MEGA_BYTES)); } - TSize FromGigaBytes(ui64 value) { - return TSize(ShiftValue(value, ESS_GIGA_BYTES)); - } + TSize FromGigaBytes(ui64 value) { + return TSize(ShiftValue(value, ESS_GIGA_BYTES)); + } - TSize FromTeraBytes(ui64 value) { - return TSize(ShiftValue(value, ESS_TERA_BYTES)); - } + TSize FromTeraBytes(ui64 value) { + return TSize(ShiftValue(value, ESS_TERA_BYTES)); + } } -template <> +template <> NSize::TSize FromStringImpl<NSize::TSize>(const char* data, size_t len) { return NSize::TSize(NSize::ParseSize(TStringBuf(data, len))); } diff --git a/library/cpp/string_utils/parse_size/parse_size.h b/library/cpp/string_utils/parse_size/parse_size.h index ad235ef02f..c9fa92980b 100644 --- a/library/cpp/string_utils/parse_size/parse_size.h +++ b/library/cpp/string_utils/parse_size/parse_size.h @@ -3,31 +3,31 @@ #include <util/generic/strbuf.h> namespace NSize { - ui64 ParseSize(TStringBuf size); - - // Convenient disk size representation with string parsing and integer comparison - class TSize { - public: - TSize(ui64 value = 0) - : Value(value) - { - } - - ui64 GetValue() const { - return Value; - } - - operator ui64() const { - return Value; - } - - private: - ui64 Value; - }; - - TSize FromKiloBytes(ui64 value); - TSize FromMegaBytes(ui64 value); - TSize FromGigaBytes(ui64 value); - TSize FromTeraBytes(ui64 value); + ui64 ParseSize(TStringBuf size); + + // Convenient disk size representation with string parsing and integer comparison + class TSize { + public: + TSize(ui64 value = 0) + : Value(value) + { + } + + ui64 GetValue() const { + return Value; + } + + operator ui64() const { + return Value; + } + + private: + ui64 Value; + }; + + TSize FromKiloBytes(ui64 value); + TSize FromMegaBytes(ui64 value); + TSize FromGigaBytes(ui64 value); + TSize FromTeraBytes(ui64 value); } diff --git a/library/cpp/string_utils/quote/quote.cpp b/library/cpp/string_utils/quote/quote.cpp index e523350b80..02bcfaa6e1 100644 --- a/library/cpp/string_utils/quote/quote.cpp +++ b/library/cpp/string_utils/quote/quote.cpp @@ -1,18 +1,18 @@ -#include "quote.h" - -#include <util/memory/tempbuf.h> +#include "quote.h" + +#include <util/memory/tempbuf.h> #include <util/string/ascii.h> #include <util/string/cstriter.h> -#include <cctype> - +#include <cctype> + /* note: (x & 0xdf) makes x upper case */ -#define GETXC \ - do { \ - c *= 16; \ - c += (x[0] >= 'A' ? ((x[0] & 0xdf) - 'A') + 10 : (x[0] - '0')); \ - ++x; \ - } while (0) +#define GETXC \ + do { \ + c *= 16; \ + c += (x[0] >= 'A' ? ((x[0] & 0xdf) - 'A') + 10 : (x[0] - '0')); \ + ++x; \ + } while (0) #define GETSBXC \ do { \ @@ -25,18 +25,18 @@ namespace { class TFromHexZeroTerm { public: - static inline char x2c(const char*& x) { - if (!IsAsciiHex((ui8)x[0]) || !IsAsciiHex((ui8)x[1])) + static inline char x2c(const char*& x) { + if (!IsAsciiHex((ui8)x[0]) || !IsAsciiHex((ui8)x[1])) return '%'; ui8 c = 0; - GETXC; - GETXC; + GETXC; + GETXC; return c; } - static inline char x2c(TStringBuf& x) { - if (!IsAsciiHex((ui8)x[0]) || !IsAsciiHex((ui8)x[1])) + static inline char x2c(TStringBuf& x) { + if (!IsAsciiHex((ui8)x[0]) || !IsAsciiHex((ui8)x[1])) return '%'; ui8 c = 0; @@ -53,7 +53,7 @@ namespace { { } - inline char x2c(const char*& x) { + inline char x2c(const char*& x) { if (x + 2 > End) return '%'; return TFromHexZeroTerm::x2c(x); @@ -64,80 +64,80 @@ namespace { }; } -static inline char d2x(unsigned x) { - return (char)((x < 10) ? ('0' + x) : ('A' + x - 10)); +static inline char d2x(unsigned x) { + return (char)((x < 10) ? ('0' + x) : ('A' + x - 10)); } static inline const char* FixZero(const char* s) noexcept { - return s ? s : ""; -} - + return s ? s : ""; +} + // we escape: -// '\"', '|', '(', ')', +// '\"', '|', '(', ')', // '%', '&', '+', ',', -// '#', '<', '=', '>', +// '#', '<', '=', '>', // '[', '\\',']', '?', // ':', '{', '}', // all below ' ' (0x20) and above '~' (0x7E). // ' ' converted to '+' static const bool chars_to_url_escape[256] = { - // 0 1 2 3 4 5 6 7 8 9 A B C D E F - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //0 - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //1 - 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, //2 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, //3 - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //4 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, //5 - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //6 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, //7 - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //8 - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //9 - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //A - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //B - - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //C - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //D - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //E - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //F + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //1 + 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, //2 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, //3 + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //4 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, //5 + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, //6 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, //7 + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //8 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //9 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //A + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //B + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //C + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //D + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //E + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, //F }; -template <class It1, class It2, class It3> -static inline It1 Escape(It1 to, It2 from, It3 end, const bool* escape_map = chars_to_url_escape) { - while (from != end) { +template <class It1, class It2, class It3> +static inline It1 Escape(It1 to, It2 from, It3 end, const bool* escape_map = chars_to_url_escape) { + while (from != end) { if (escape_map[(unsigned char)*from]) { *to++ = '%'; - *to++ = d2x((unsigned char)*from >> 4); - *to++ = d2x((unsigned char)*from & 0xF); + *to++ = d2x((unsigned char)*from >> 4); + *to++ = d2x((unsigned char)*from & 0xF); } else { *to++ = (*from == ' ' ? '+' : *from); } - + ++from; } - + *to = 0; - + return to; } template <class It1, class It2, class It3, class FromHex> static inline It1 Unescape(It1 to, It2 from, It3 end, FromHex fromHex) { - (void)fromHex; - + (void)fromHex; + while (from != end) { switch (*from) { - case '%': + case '%': ++from; - *to++ = fromHex.x2c(from); - break; - case '+': - *to++ = ' '; + *to++ = fromHex.x2c(from); + break; + case '+': + *to++ = ' '; ++from; - break; - default: - *to++ = *from++; + break; + default: + *to++ = *from++; } } *to = 0; @@ -147,18 +147,18 @@ static inline It1 Unescape(It1 to, It2 from, It3 end, FromHex fromHex) { // CGIEscape returns pointer to the end of the result string // so as it could be possible to populate single long buffer // with several calls to CGIEscape in a row. -char* CGIEscape(char* to, const char* from) { - return Escape(to, FixZero(from), TCStringEndIterator()); -} - -char* CGIEscape(char* to, const char* from, size_t len) { - return Escape(to, from, from + len); +char* CGIEscape(char* to, const char* from) { + return Escape(to, FixZero(from), TCStringEndIterator()); } +char* CGIEscape(char* to, const char* from, size_t len) { + return Escape(to, from, from + len); +} + void CGIEscape(TString& url) { TTempBuf tempBuf(CgiEscapeBufLen(url.size())); - char* to = tempBuf.Data(); - + char* to = tempBuf.Data(); + url.AssignNoAlias(to, CGIEscape(to, url.data(), url.size())); } @@ -178,7 +178,7 @@ TString& AppendCgiEscaped(const TStringBuf value, TString& to) { // More general version of CGIEscape. The optional safe parameter specifies // additional characters that should not be quoted — its default value is '/'. - + // Also returns pointer to the end of result string. template <class It1, class It2, class It3> @@ -210,14 +210,14 @@ char* Quote(char* to, const TStringBuf s, const char* safe) { void Quote(TString& url, const char* safe) { TTempBuf tempBuf(CgiEscapeBufLen(url.size())); char* to = tempBuf.Data(); - + url.AssignNoAlias(to, Quote(to, url, safe)); } -char* CGIUnescape(char* to, const char* from) { +char* CGIUnescape(char* to, const char* from) { return Unescape(to, FixZero(from), TCStringEndIterator(), TFromHexZeroTerm()); } - + char* CGIUnescape(char* to, const char* from, size_t len) { return Unescape(to, from, from + len, TFromHexLenLimited(from + len)); } @@ -250,10 +250,10 @@ char* UrlUnescape(char* to, TStringBuf from) { ch = TFromHexZeroTerm::x2c(from); *to++ = ch; } - + *to = 0; - - return to; + + return to; } void UrlUnescape(TString& url) { @@ -277,29 +277,29 @@ TString UrlUnescapeRet(const TStringBuf from) { } char* UrlEscape(char* to, const char* from, bool forceEscape) { - from = FixZero(from); - - while (*from) { + from = FixZero(from); + + while (*from) { const bool escapePercent = (*from == '%') && (forceEscape || !((*(from + 1) && IsAsciiHex(*(from + 1)) && *(from + 2) && IsAsciiHex(*(from + 2))))); if (escapePercent || (unsigned char)*from <= ' ' || (unsigned char)*from > '~') { - *to++ = '%'; - *to++ = d2x((unsigned char)*from >> 4); - *to++ = d2x((unsigned char)*from & 0xF); - } else + *to++ = '%'; + *to++ = d2x((unsigned char)*from >> 4); + *to++ = d2x((unsigned char)*from & 0xF); + } else *to++ = *from; ++from; } - + *to = 0; - - return to; + + return to; } void UrlEscape(TString& url, bool forceEscape) { TTempBuf tempBuf(CgiEscapeBufLen(url.size())); - char* to = tempBuf.Data(); + char* to = tempBuf.Data(); url.AssignNoAlias(to, UrlEscape(to, url.data(), forceEscape)); } diff --git a/library/cpp/string_utils/quote/quote.h b/library/cpp/string_utils/quote/quote.h index 3b7221154e..29c2bf7f89 100644 --- a/library/cpp/string_utils/quote/quote.h +++ b/library/cpp/string_utils/quote/quote.h @@ -1,29 +1,29 @@ -#pragma once - -#include <util/generic/strbuf.h> +#pragma once + +#include <util/generic/strbuf.h> #include <util/generic/string.h> - + //CGIEscape*: // ' ' converted to '+', // Some punctuation and chars outside [32, 126] range are converted to %xx // Use function CgiEscapeBufLen to determine number of characters needed for 'char* to' parameter. // Returns pointer to the end of the result string -char* CGIEscape(char* to, const char* from); -char* CGIEscape(char* to, const char* from, size_t len); +char* CGIEscape(char* to, const char* from); +char* CGIEscape(char* to, const char* from, size_t len); inline char* CGIEscape(char* to, const TStringBuf from) { return CGIEscape(to, from.data(), from.size()); } void CGIEscape(TString& url); TString CGIEscapeRet(const TStringBuf url); TString& AppendCgiEscaped(const TStringBuf value, TString& to); - + inline TStringBuf CgiEscapeBuf(char* to, const TStringBuf from) { return TStringBuf(to, CGIEscape(to, from.data(), from.size())); } inline TStringBuf CgiEscape(void* tmp, const TStringBuf s) { - return CgiEscapeBuf(static_cast<char*>(tmp), s); -} - + return CgiEscapeBuf(static_cast<char*>(tmp), s); +} + //CgiUnescape*: // Decodes '%xx' to bytes, '+' to space. // Use function CgiUnescapeBufLen to determine number of characters needed for 'char* to' parameter. @@ -37,15 +37,15 @@ inline TStringBuf CgiUnescapeBuf(char* to, const TStringBuf from) { return TStringBuf(to, CGIUnescape(to, from.data(), from.size())); } inline TStringBuf CgiUnescape(void* tmp, const TStringBuf s) { - return CgiUnescapeBuf(static_cast<char*>(tmp), s); -} - + return CgiUnescapeBuf(static_cast<char*>(tmp), s); +} + //Quote: // Is like CGIEscape, also skips encoding of user-supplied 'safe' characters. -char* Quote(char* to, const char* from, const char* safe = "/"); +char* Quote(char* to, const char* from, const char* safe = "/"); char* Quote(char* to, const TStringBuf s, const char* safe = "/"); void Quote(TString& url, const char* safe = "/"); - + //UrlEscape: // Can't be used for cgi parameters ('&' character is not escaped)! // escapes only '%' not followed by two hex-digits or if forceEscape set to ture, @@ -61,12 +61,12 @@ TString UrlEscapeRet(const TStringBuf from, bool forceEscape = false); char* UrlUnescape(char* to, TStringBuf from); void UrlUnescape(TString& url); TString UrlUnescapeRet(const TStringBuf from); - + //*BufLen: how much characters you should allocate for 'char* to' buffers. constexpr size_t CgiEscapeBufLen(const size_t len) noexcept { - return 3 * len + 1; -} - + return 3 * len + 1; +} + constexpr size_t CgiUnescapeBufLen(const size_t len) noexcept { - return len + 1; -} + return len + 1; +} diff --git a/library/cpp/string_utils/quote/quote_ut.cpp b/library/cpp/string_utils/quote/quote_ut.cpp index 6c552b279e..b89a9525a2 100644 --- a/library/cpp/string_utils/quote/quote_ut.cpp +++ b/library/cpp/string_utils/quote/quote_ut.cpp @@ -1,7 +1,7 @@ -#include "quote.h" - +#include "quote.h" + #include <library/cpp/testing/unittest/registar.h> - + Y_UNIT_TEST_SUITE(TCGIEscapeTest) { Y_UNIT_TEST(ReturnsEndOfTo) { char r[10]; @@ -9,21 +9,21 @@ Y_UNIT_TEST_SUITE(TCGIEscapeTest) { UNIT_ASSERT_VALUES_EQUAL(r + strlen("123"), returned); UNIT_ASSERT_VALUES_EQUAL('\0', *returned); } - + Y_UNIT_TEST(NotZeroTerminated) { - char r[] = {'1', '2', '3', '4'}; - char buf[sizeof(r) * 3 + 2]; - + char r[] = {'1', '2', '3', '4'}; + char buf[sizeof(r) * 3 + 2]; + TString ret(buf, CGIEscape(buf, r, sizeof(r))); - - UNIT_ASSERT_EQUAL(ret, "1234"); - } - + + UNIT_ASSERT_EQUAL(ret, "1234"); + } + Y_UNIT_TEST(StringBuf) { - char tmp[100]; - + char tmp[100]; + UNIT_ASSERT_VALUES_EQUAL(CgiEscape(tmp, "!@#$%^&*(){}[]\" "), TStringBuf("!@%23$%25^%26*%28%29%7B%7D%5B%5D%22+")); - } + } Y_UNIT_TEST(StrokaRet) { UNIT_ASSERT_VALUES_EQUAL(CGIEscapeRet("!@#$%^&*(){}[]\" "), TString("!@%23$%25^%26*%28%29%7B%7D%5B%5D%22+")); @@ -49,11 +49,11 @@ Y_UNIT_TEST_SUITE(TCGIEscapeTest) { Y_UNIT_TEST_SUITE(TCGIUnescapeTest) { Y_UNIT_TEST(StringBuf) { - char tmp[100]; - + char tmp[100]; + UNIT_ASSERT_VALUES_EQUAL(CgiUnescape(tmp, "!@%23$%25^%26*%28%29"), TStringBuf("!@#$%^&*()")); - } - + } + Y_UNIT_TEST(TestValidZeroTerm) { char r[10]; diff --git a/library/cpp/string_utils/relaxed_escaper/relaxed_escaper.cpp b/library/cpp/string_utils/relaxed_escaper/relaxed_escaper.cpp index ac624dca85..8a04323fac 100644 --- a/library/cpp/string_utils/relaxed_escaper/relaxed_escaper.cpp +++ b/library/cpp/string_utils/relaxed_escaper/relaxed_escaper.cpp @@ -1 +1 @@ -#include "relaxed_escaper.h" +#include "relaxed_escaper.h" diff --git a/library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h b/library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h index d7ea7c1259..999a75b601 100644 --- a/library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h +++ b/library/cpp/string_utils/relaxed_escaper/relaxed_escaper.h @@ -2,207 +2,207 @@ #include <util/stream/output.h> #include <util/string/escape.h> -#include <util/memory/tempbuf.h> -#include <util/generic/strbuf.h> +#include <util/memory/tempbuf.h> +#include <util/generic/strbuf.h> namespace NEscJ { - // almost copypaste from util/string/escape.h - // todo: move there (note difference in IsPrintable and handling of string) - - inline char HexDigit(char value) { - if (value < 10) - return '0' + value; - else - return 'A' + value - 10; - } - - inline char OctDigit(char value) { + // almost copypaste from util/string/escape.h + // todo: move there (note difference in IsPrintable and handling of string) + + inline char HexDigit(char value) { + if (value < 10) + return '0' + value; + else + return 'A' + value - 10; + } + + inline char OctDigit(char value) { return '0' + value; - } - - inline bool IsUTF8(ui8 c) { - return c < 0xf5 && c != 0xC0 && c != 0xC1; - } - - inline bool IsControl(ui8 c) { - return c < 0x20 || c == 0x7f; - } - - inline bool IsPrintable(ui8 c) { - return IsUTF8(c) && !IsControl(c); - } - - inline bool IsHexDigit(ui8 c) { - return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); - } - - inline bool IsOctDigit(ui8 c) { - return c >= '0' && c <= '7'; - } - - struct TEscapeUtil { - static const size_t ESCAPE_C_BUFFER_SIZE = 6; - - template <bool asunicode> - static inline size_t EscapeJ(ui8 c, ui8 next, char r[ESCAPE_C_BUFFER_SIZE], TStringBuf safe, TStringBuf unsafe) { - // (1) Printable characters go as-is, except backslash and double quote. - // (2) Characters \r, \n, \t and \0 ... \7 replaced by their simple escape characters (if possible). - // (3) Otherwise, character is encoded using hexadecimal escape sequence (if possible), or octal. - if (safe.find(c) != TStringBuf::npos) { - r[0] = c; - return 1; - } - if (c == '\"') { - r[0] = '\\'; - r[1] = '\"'; - return 2; - } else if (c == '\\') { - r[0] = '\\'; - r[1] = '\\'; - return 2; - } else if (IsPrintable(c) && unsafe.find(c) == TStringBuf::npos) { - r[0] = c; - return 1; - } else if (c == '\b') { - r[0] = '\\'; - r[1] = 'b'; - return 2; - } else if (c == '\f') { - r[0] = '\\'; - r[1] = 'f'; - return 2; - } else if (c == '\r') { - r[0] = '\\'; - r[1] = 'r'; - return 2; - } else if (c == '\n') { - r[0] = '\\'; - r[1] = 'n'; - return 2; - } else if (c == '\t') { - r[0] = '\\'; - r[1] = 't'; - return 2; - } else if (asunicode && IsUTF8(c)) { // utf8 controls escape for json - r[0] = '\\'; - r[1] = 'u'; - r[2] = '0'; - r[3] = '0'; - r[4] = HexDigit((c & 0xF0) >> 4); - r[5] = HexDigit((c & 0x0F) >> 0); - return 6; - } else if (c < 8 && !IsOctDigit(next)) { - r[0] = '\\'; - r[1] = OctDigit(c); - return 2; - } else if (!IsHexDigit(next)) { - r[0] = '\\'; - r[1] = 'x'; - r[2] = HexDigit((c & 0xF0) >> 4); - r[3] = HexDigit((c & 0x0F) >> 0); - return 4; - } else { - r[0] = '\\'; - r[1] = OctDigit((c & 0700) >> 6); - r[2] = OctDigit((c & 0070) >> 3); - r[3] = OctDigit((c & 0007) >> 0); - return 4; - } - } - - static inline size_t EscapeJ(ui8 c, ui8 next, char r[ESCAPE_C_BUFFER_SIZE], TStringBuf safe, TStringBuf unsafe) { - return EscapeJ<false>(c, next, r, safe, unsafe); + } + + inline bool IsUTF8(ui8 c) { + return c < 0xf5 && c != 0xC0 && c != 0xC1; + } + + inline bool IsControl(ui8 c) { + return c < 0x20 || c == 0x7f; + } + + inline bool IsPrintable(ui8 c) { + return IsUTF8(c) && !IsControl(c); + } + + inline bool IsHexDigit(ui8 c) { + return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); + } + + inline bool IsOctDigit(ui8 c) { + return c >= '0' && c <= '7'; + } + + struct TEscapeUtil { + static const size_t ESCAPE_C_BUFFER_SIZE = 6; + + template <bool asunicode> + static inline size_t EscapeJ(ui8 c, ui8 next, char r[ESCAPE_C_BUFFER_SIZE], TStringBuf safe, TStringBuf unsafe) { + // (1) Printable characters go as-is, except backslash and double quote. + // (2) Characters \r, \n, \t and \0 ... \7 replaced by their simple escape characters (if possible). + // (3) Otherwise, character is encoded using hexadecimal escape sequence (if possible), or octal. + if (safe.find(c) != TStringBuf::npos) { + r[0] = c; + return 1; + } + if (c == '\"') { + r[0] = '\\'; + r[1] = '\"'; + return 2; + } else if (c == '\\') { + r[0] = '\\'; + r[1] = '\\'; + return 2; + } else if (IsPrintable(c) && unsafe.find(c) == TStringBuf::npos) { + r[0] = c; + return 1; + } else if (c == '\b') { + r[0] = '\\'; + r[1] = 'b'; + return 2; + } else if (c == '\f') { + r[0] = '\\'; + r[1] = 'f'; + return 2; + } else if (c == '\r') { + r[0] = '\\'; + r[1] = 'r'; + return 2; + } else if (c == '\n') { + r[0] = '\\'; + r[1] = 'n'; + return 2; + } else if (c == '\t') { + r[0] = '\\'; + r[1] = 't'; + return 2; + } else if (asunicode && IsUTF8(c)) { // utf8 controls escape for json + r[0] = '\\'; + r[1] = 'u'; + r[2] = '0'; + r[3] = '0'; + r[4] = HexDigit((c & 0xF0) >> 4); + r[5] = HexDigit((c & 0x0F) >> 0); + return 6; + } else if (c < 8 && !IsOctDigit(next)) { + r[0] = '\\'; + r[1] = OctDigit(c); + return 2; + } else if (!IsHexDigit(next)) { + r[0] = '\\'; + r[1] = 'x'; + r[2] = HexDigit((c & 0xF0) >> 4); + r[3] = HexDigit((c & 0x0F) >> 0); + return 4; + } else { + r[0] = '\\'; + r[1] = OctDigit((c & 0700) >> 6); + r[2] = OctDigit((c & 0070) >> 3); + r[3] = OctDigit((c & 0007) >> 0); + return 4; + } + } + + static inline size_t EscapeJ(ui8 c, ui8 next, char r[ESCAPE_C_BUFFER_SIZE], TStringBuf safe, TStringBuf unsafe) { + return EscapeJ<false>(c, next, r, safe, unsafe); } - }; + }; - inline size_t SuggestBuffer(size_t len) { - return len * TEscapeUtil::ESCAPE_C_BUFFER_SIZE; + inline size_t SuggestBuffer(size_t len) { + return len * TEscapeUtil::ESCAPE_C_BUFFER_SIZE; } - template <bool tounicode> - inline size_t EscapeJ(const char* str, size_t len, char* out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { - char* out0 = out; - char buffer[TEscapeUtil::ESCAPE_C_BUFFER_SIZE]; + template <bool tounicode> + inline size_t EscapeJ(const char* str, size_t len, char* out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { + char* out0 = out; + char buffer[TEscapeUtil::ESCAPE_C_BUFFER_SIZE]; - size_t i, j; - for (i = 0, j = 0; i < len; ++i) { - size_t rlen = TEscapeUtil::EscapeJ<tounicode>(str[i], (i + 1 < len ? str[i + 1] : 0), buffer, safe, unsafe); + size_t i, j; + for (i = 0, j = 0; i < len; ++i) { + size_t rlen = TEscapeUtil::EscapeJ<tounicode>(str[i], (i + 1 < len ? str[i + 1] : 0), buffer, safe, unsafe); - if (rlen > 1) { - strncpy(out, str + j, i - j); - out += i - j; - j = i + 1; + if (rlen > 1) { + strncpy(out, str + j, i - j); + out += i - j; + j = i + 1; - strncpy(out, buffer, rlen); - out += rlen; - } - } + strncpy(out, buffer, rlen); + out += rlen; + } + } - if (j > 0) { - strncpy(out, str + j, len - j); - out += len - j; - } else { - strncpy(out, str, len); - out += len; + if (j > 0) { + strncpy(out, str + j, len - j); + out += len - j; + } else { + strncpy(out, str, len); + out += len; } - return out - out0; + return out - out0; } - template <bool quote, bool tounicode> + template <bool quote, bool tounicode> inline void EscapeJ(TStringBuf in, IOutputStream& out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { TTempBuf b(SuggestBuffer(in.size()) + 2); - if (quote) - b.Append("\"", 1); + if (quote) + b.Append("\"", 1); b.Proceed(EscapeJ<tounicode>(in.data(), in.size(), b.Current(), safe, unsafe)); - if (quote) - b.Append("\"", 1); + if (quote) + b.Append("\"", 1); - out.Write(b.Data(), b.Filled()); - } + out.Write(b.Data(), b.Filled()); + } - template <bool quote, bool tounicode> + template <bool quote, bool tounicode> inline void EscapeJ(TStringBuf in, TString& out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { TTempBuf b(SuggestBuffer(in.size()) + 2); - if (quote) - b.Append("\"", 1); + if (quote) + b.Append("\"", 1); b.Proceed(EscapeJ<tounicode>(in.data(), in.size(), b.Current(), safe, unsafe)); - if (quote) - b.Append("\"", 1); + if (quote) + b.Append("\"", 1); - out.append(b.Data(), b.Filled()); - } + out.append(b.Data(), b.Filled()); + } - template <bool quote, bool tounicode> + template <bool quote, bool tounicode> inline TString EscapeJ(TStringBuf in, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { TString s; - EscapeJ<quote, tounicode>(in, s, safe, unsafe); - return s; - } + EscapeJ<quote, tounicode>(in, s, safe, unsafe); + return s; + } - // If the template parameter "tounicode" is ommited, then use the default value false - inline size_t EscapeJ(const char* str, size_t len, char* out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { - return EscapeJ<false>(str, len, out, safe, unsafe); - } + // If the template parameter "tounicode" is ommited, then use the default value false + inline size_t EscapeJ(const char* str, size_t len, char* out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { + return EscapeJ<false>(str, len, out, safe, unsafe); + } - template <bool quote> + template <bool quote> inline void EscapeJ(TStringBuf in, IOutputStream& out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { - EscapeJ<quote, false>(in, out, safe, unsafe); - } + EscapeJ<quote, false>(in, out, safe, unsafe); + } - template <bool quote> + template <bool quote> inline void EscapeJ(TStringBuf in, TString& out, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { - EscapeJ<quote, false>(in, out, safe, unsafe); - } + EscapeJ<quote, false>(in, out, safe, unsafe); + } - template <bool quote> + template <bool quote> inline TString EscapeJ(TStringBuf in, TStringBuf safe = TStringBuf(), TStringBuf unsafe = TStringBuf()) { - return EscapeJ<quote, false>(in, safe, unsafe); - } + return EscapeJ<quote, false>(in, safe, unsafe); + } } diff --git a/library/cpp/string_utils/relaxed_escaper/relaxed_escaper_ut.cpp b/library/cpp/string_utils/relaxed_escaper/relaxed_escaper_ut.cpp index 768555ea3a..496c30ab01 100644 --- a/library/cpp/string_utils/relaxed_escaper/relaxed_escaper_ut.cpp +++ b/library/cpp/string_utils/relaxed_escaper/relaxed_escaper_ut.cpp @@ -10,19 +10,19 @@ static const TStringBuf CommonTestData[] = { RESC_FIXED_STR("http://ya.ru/\\0"), RESC_FIXED_STR("http://ya.ru/\0"), RESC_FIXED_STR("http://ya.ru/\\0\\0"), RESC_FIXED_STR("http://ya.ru/\0\0"), - RESC_FIXED_STR("http://ya.ru/\\0\\0000"), RESC_FIXED_STR("http://ya.ru/\0\0" - "0"), - RESC_FIXED_STR("http://ya.ru/\\0\\0001"), RESC_FIXED_STR("http://ya.ru/\0\x00" - "1"), + RESC_FIXED_STR("http://ya.ru/\\0\\0000"), RESC_FIXED_STR("http://ya.ru/\0\0" + "0"), + RESC_FIXED_STR("http://ya.ru/\\0\\0001"), RESC_FIXED_STR("http://ya.ru/\0\x00" + "1"), - RESC_FIXED_STR("\\2\\4\\00678"), RESC_FIXED_STR("\2\4\6" - "78"), + RESC_FIXED_STR("\\2\\4\\00678"), RESC_FIXED_STR("\2\4\6" + "78"), RESC_FIXED_STR("\\2\\4\\689"), RESC_FIXED_STR("\2\4\689"), RESC_FIXED_STR("\\\"Hello\\\", Alice said."), RESC_FIXED_STR("\"Hello\", Alice said."), RESC_FIXED_STR("Slash\\\\dash!"), RESC_FIXED_STR("Slash\\dash!"), RESC_FIXED_STR("There\\nare\\r\\nnewlines."), RESC_FIXED_STR("There\nare\r\nnewlines."), - RESC_FIXED_STR("There\\tare\\ttabs."), RESC_FIXED_STR("There\tare\ttabs.")}; + RESC_FIXED_STR("There\\tare\\ttabs."), RESC_FIXED_STR("There\tare\ttabs.")}; #undef RESC_FIXED_STR Y_UNIT_TEST_SUITE(TRelaxedEscaperTest) { @@ -52,15 +52,15 @@ Y_UNIT_TEST_SUITE(TRelaxedEscaperTest) { UNIT_ASSERT_VALUES_EQUAL("\"\\xFF\"", EscapeJ<true>("\xFF")); UNIT_ASSERT_VALUES_EQUAL("\xFF", UnescapeC("\\xFF")); - UNIT_ASSERT_VALUES_EQUAL("\\377f", EscapeJ<false>("\xff" - "f")); - UNIT_ASSERT_VALUES_EQUAL("\xff" - "f", - UnescapeC("\\377f")); - UNIT_ASSERT_VALUES_EQUAL("\\xFFg", EscapeJ<false>("\xff" - "g")); - UNIT_ASSERT_VALUES_EQUAL("\xff" - "g", - UnescapeC("\\xFFg")); + UNIT_ASSERT_VALUES_EQUAL("\\377f", EscapeJ<false>("\xff" + "f")); + UNIT_ASSERT_VALUES_EQUAL("\xff" + "f", + UnescapeC("\\377f")); + UNIT_ASSERT_VALUES_EQUAL("\\xFFg", EscapeJ<false>("\xff" + "g")); + UNIT_ASSERT_VALUES_EQUAL("\xff" + "g", + UnescapeC("\\xFFg")); } } diff --git a/library/cpp/string_utils/relaxed_escaper/ut/ya.make b/library/cpp/string_utils/relaxed_escaper/ut/ya.make index 7ebd393c48..55a9611de1 100644 --- a/library/cpp/string_utils/relaxed_escaper/ut/ya.make +++ b/library/cpp/string_utils/relaxed_escaper/ut/ya.make @@ -1,9 +1,9 @@ UNITTEST_FOR(library/cpp/string_utils/relaxed_escaper) - -OWNER(velavokr) - -SRCS( - relaxed_escaper_ut.cpp -) - -END() + +OWNER(velavokr) + +SRCS( + relaxed_escaper_ut.cpp +) + +END() diff --git a/library/cpp/string_utils/relaxed_escaper/ya.make b/library/cpp/string_utils/relaxed_escaper/ya.make index 3f0fa5bc07..4e0364c633 100644 --- a/library/cpp/string_utils/relaxed_escaper/ya.make +++ b/library/cpp/string_utils/relaxed_escaper/ya.make @@ -1,9 +1,9 @@ -LIBRARY() - -OWNER(velavokr) - -SRCS( - relaxed_escaper.cpp -) - -END() +LIBRARY() + +OWNER(velavokr) + +SRCS( + relaxed_escaper.cpp +) + +END() diff --git a/library/cpp/string_utils/scan/scan.cpp b/library/cpp/string_utils/scan/scan.cpp index fbc1fdf08f..5e418a0cb8 100644 --- a/library/cpp/string_utils/scan/scan.cpp +++ b/library/cpp/string_utils/scan/scan.cpp @@ -1 +1 @@ -#include "scan.h" +#include "scan.h" diff --git a/library/cpp/string_utils/scan/scan.h b/library/cpp/string_utils/scan/scan.h index 703db54321..466ea46537 100644 --- a/library/cpp/string_utils/scan/scan.h +++ b/library/cpp/string_utils/scan/scan.h @@ -1,22 +1,22 @@ -#pragma once - -#include <util/generic/strbuf.h> - +#pragma once + +#include <util/generic/strbuf.h> + template <bool addAll, char sep, char sepKeyVal, class F> -static inline void ScanKeyValue(TStringBuf s, F&& f) { +static inline void ScanKeyValue(TStringBuf s, F&& f) { TStringBuf key, val; - + while (!s.empty()) { val = s.NextTok(sep); - + if (val.empty()) { continue; // && case - } - + } + key = val.NextTok(sepKeyVal); - - if (addAll || val.IsInited()) { + + if (addAll || val.IsInited()) { f(key, val); // includes empty keys - } - } -} + } + } +} diff --git a/library/cpp/string_utils/url/url.cpp b/library/cpp/string_utils/url/url.cpp index 85f4ac5d69..0e9e482ed6 100644 --- a/library/cpp/string_utils/url/url.cpp +++ b/library/cpp/string_utils/url/url.cpp @@ -1,5 +1,5 @@ -#include "url.h" - +#include "url.h" + #include <util/string/cast.h> #include <util/string/util.h> #include <util/string/cstriter.h> @@ -11,7 +11,7 @@ #include <util/generic/algorithm.h> #include <util/generic/hash_set.h> #include <util/generic/yexception.h> -#include <util/generic/singleton.h> +#include <util/generic/singleton.h> #include <cstdlib> @@ -26,8 +26,8 @@ namespace { size_t MySize; explicit TKnownSize(size_t sz) : MySize(sz) - { - } + { + } bool Has(size_t sz) const { return sz <= MySize; } @@ -43,9 +43,9 @@ namespace { } template <typename TChar, typename TBounds> - inline size_t GetHttpPrefixSizeImpl(const TChar* url, const TBounds& urlSize, bool ignorehttps) { - const TChar httpPrefix[] = {'h', 't', 't', 'p', ':', '/', '/', 0}; - const TChar httpsPrefix[] = {'h', 't', 't', 'p', 's', ':', '/', '/', 0}; + inline size_t GetHttpPrefixSizeImpl(const TChar* url, const TBounds& urlSize, bool ignorehttps) { + const TChar httpPrefix[] = {'h', 't', 't', 'p', ':', '/', '/', 0}; + const TChar httpsPrefix[] = {'h', 't', 't', 'p', 's', ':', '/', '/', 0}; if (urlSize.Has(7) && Compare1Case2(url, httpPrefix, 7) == 0) return 7; if (!ignorehttps && urlSize.Has(8) && Compare1Case2(url, httpsPrefix, 8) == 0) @@ -98,20 +98,20 @@ TWtringBuf CutHttpPrefix(const TWtringBuf url, bool ignorehttps) noexcept { } size_t GetSchemePrefixSize(const TStringBuf url) noexcept { - struct TDelim: public str_spn { - inline TDelim() - : str_spn("!-/:-@[-`{|}", true) - { - } - }; - - const auto& delim = *Singleton<TDelim>(); + struct TDelim: public str_spn { + inline TDelim() + : str_spn("!-/:-@[-`{|}", true) + { + } + }; + + const auto& delim = *Singleton<TDelim>(); const char* n = delim.brk(url.data(), url.end()); - - if (n + 2 >= url.end() || *n != ':' || n[1] != '/' || n[2] != '/') { + + if (n + 2 >= url.end() || *n != ':' || n[1] != '/' || n[2] != '/') { return 0; - } - + } + return n + 3 - url.begin(); } @@ -123,27 +123,27 @@ TStringBuf CutSchemePrefix(const TStringBuf url) noexcept { return url.Tail(GetSchemePrefixSize(url)); } -template <bool KeepPort> +template <bool KeepPort> static inline TStringBuf GetHostAndPortImpl(const TStringBuf url) { TStringBuf urlNoScheme = url; - + urlNoScheme.Skip(GetHttpPrefixSize(url)); - struct TDelim: public str_spn { - inline TDelim() - : str_spn(KeepPort ? "/;?#" : "/:;?#") - { - } - }; - - const auto& nonHostCharacters = *Singleton<TDelim>(); + struct TDelim: public str_spn { + inline TDelim() + : str_spn(KeepPort ? "/;?#" : "/:;?#") + { + } + }; + + const auto& nonHostCharacters = *Singleton<TDelim>(); const char* firstNonHostCharacter = nonHostCharacters.brk(urlNoScheme.begin(), urlNoScheme.end()); - - if (firstNonHostCharacter != urlNoScheme.end()) { + + if (firstNonHostCharacter != urlNoScheme.end()) { return urlNoScheme.substr(0, firstNonHostCharacter - urlNoScheme.data()); - } - - return urlNoScheme; + } + + return urlNoScheme; } TStringBuf GetHost(const TStringBuf url) noexcept { @@ -337,27 +337,27 @@ TString AddSchemePrefix(const TString& url, TStringBuf scheme) { #define X(c) (c >= 'A' ? ((c & 0xdf) - 'A') + 10 : (c - '0')) static inline int x2c(unsigned char* x) { - if (!IsAsciiHex(x[0]) || !IsAsciiHex(x[1])) + if (!IsAsciiHex(x[0]) || !IsAsciiHex(x[1])) return -1; return X(x[0]) * 16 + X(x[1]); } #undef X -static inline int Unescape(char* str) { - char *to, *from; +static inline int Unescape(char* str) { + char *to, *from; int dlen = 0; if ((str = strchr(str, '%')) == nullptr) return dlen; for (to = str, from = str; *from; from++, to++) { if ((*to = *from) == '%') { - int c = x2c((unsigned char*)from + 1); + int c = x2c((unsigned char*)from + 1); *to = char((c > 0) ? c : '0'); from += 2; dlen += 2; } } - *to = 0; /* terminate it at the new length */ + *to = 0; /* terminate it at the new length */ return dlen; } @@ -379,8 +379,8 @@ size_t NormalizeHostName(char* dest, const TStringBuf source, size_t dest_size, char buf[8] = ":"; size_t buflen = 1 + ToString(defport, buf + 1, sizeof(buf) - 2); buf[buflen] = '\0'; - char* ptr = strstr(dest, buf); - if (ptr && ptr[buflen] == 0) { + char* ptr = strstr(dest, buf); + if (ptr && ptr[buflen] == 0) { len -= buflen; *ptr = 0; } diff --git a/library/cpp/string_utils/url/url_ut.cpp b/library/cpp/string_utils/url/url_ut.cpp index 1588013893..bf12597c98 100644 --- a/library/cpp/string_utils/url/url_ut.cpp +++ b/library/cpp/string_utils/url/url_ut.cpp @@ -1,5 +1,5 @@ -#include "url.h" - +#include "url.h" + #include <util/string/cast.h> #include <library/cpp/testing/unittest/registar.h> @@ -100,7 +100,7 @@ Y_UNIT_TEST_SUITE(TUtilUrlTest) { UNIT_ASSERT_VALUES_EQUAL("ya.ru/zzz", CutHttpPrefix("http://ya.ru/zzz", true)); UNIT_ASSERT_VALUES_EQUAL("ya.ru/zzz", CutHttpPrefix("https://ya.ru/zzz")); UNIT_ASSERT_VALUES_EQUAL("https://ya.ru/zzz", CutHttpPrefix("https://ya.ru/zzz", true)); - UNIT_ASSERT_VALUES_EQUAL("", CutHttpPrefix("https://")); // is that right? + UNIT_ASSERT_VALUES_EQUAL("", CutHttpPrefix("https://")); // is that right? UNIT_ASSERT_VALUES_EQUAL("https://", CutHttpPrefix("https://", true)); // is that right? } diff --git a/library/cpp/svnversion/svnversion.cpp b/library/cpp/svnversion/svnversion.cpp index 4c9761fa75..1e9fc97668 100644 --- a/library/cpp/svnversion/svnversion.cpp +++ b/library/cpp/svnversion/svnversion.cpp @@ -2,8 +2,8 @@ #include <stdlib.h> #include <stdio.h> -#define FROM_IMPL -#include "svnversion.h" +#define FROM_IMPL +#include "svnversion.h" #include <util/generic/strbuf.h> @@ -16,18 +16,18 @@ extern "C" void PrintSvnVersionAndExit0() { exit(0); } -extern "C" void PrintSvnVersionAndExitEx(int argc, char* argv[], const char* opts) { +extern "C" void PrintSvnVersionAndExitEx(int argc, char* argv[], const char* opts) { if (2 == argc) { - for (TStringBuf all = opts, versionOpt; all.NextTok(';', versionOpt);) { + for (TStringBuf all = opts, versionOpt; all.NextTok(';', versionOpt);) { if (versionOpt == argv[1]) { PrintSvnVersionAndExit0(); } } } } - -extern "C" void PrintSvnVersionAndExit(int argc, char* argv[]) { + +extern "C" void PrintSvnVersionAndExit(int argc, char* argv[]) { PrintSvnVersionAndExitEx(argc, argv, "--version"); } -#undef FROM_IMPL +#undef FROM_IMPL diff --git a/library/cpp/svnversion/svnversion.h b/library/cpp/svnversion/svnversion.h index b99615daa9..2ee308f91b 100644 --- a/library/cpp/svnversion/svnversion.h +++ b/library/cpp/svnversion/svnversion.h @@ -1,11 +1,11 @@ #pragma once -#if !defined(FROM_IMPL) +#if !defined(FROM_IMPL) #define PROGRAM_VERSION GetProgramSvnVersion() #define ARCADIA_SOURCE_PATH GetArcadiaSourcePath() #define PRINT_VERSION PrintSvnVersionAndExit(argc, (char**)argv) #define PRINT_VERSION_EX(opts) PrintSvnVersionAndExitEx(argc, (char**)argv, opts) -#endif +#endif #include <util/system/compiler.h> diff --git a/library/cpp/svnversion/ya.make b/library/cpp/svnversion/ya.make index 04e1a4c4fd..146a3e9408 100644 --- a/library/cpp/svnversion/ya.make +++ b/library/cpp/svnversion/ya.make @@ -1,6 +1,6 @@ OWNER(g:cpp-contrib) -LIBRARY() +LIBRARY() SRCS( svnversion.cpp diff --git a/library/cpp/terminate_handler/sample/pure-virtual/main.cpp b/library/cpp/terminate_handler/sample/pure-virtual/main.cpp index 58217d5f24..d3b2c51d35 100644 --- a/library/cpp/terminate_handler/sample/pure-virtual/main.cpp +++ b/library/cpp/terminate_handler/sample/pure-virtual/main.cpp @@ -12,8 +12,8 @@ struct TFoo { }; struct TQux: public TFoo { - void Bar() override { - } + void Bar() override { + } }; int main() { diff --git a/library/cpp/terminate_handler/sample/segv/main.cpp b/library/cpp/terminate_handler/sample/segv/main.cpp index 52851bdb19..61479edc1c 100644 --- a/library/cpp/terminate_handler/sample/segv/main.cpp +++ b/library/cpp/terminate_handler/sample/segv/main.cpp @@ -10,6 +10,6 @@ void Foo(int* x) { int main() { InstallSegvHandler(); - Foo((int*)1); + Foo((int*)1); return 0; } diff --git a/library/cpp/testing/benchmark/bench.cpp b/library/cpp/testing/benchmark/bench.cpp index 08d8708005..be9bfbad3b 100644 --- a/library/cpp/testing/benchmark/bench.cpp +++ b/library/cpp/testing/benchmark/bench.cpp @@ -1,5 +1,5 @@ -#include "bench.h" - +#include "bench.h" + #include <contrib/libs/re2/re2/re2.h> #include <library/cpp/colorizer/output.h> @@ -7,287 +7,287 @@ #include <library/cpp/json/json_value.h> #include <library/cpp/linear_regression/linear_regression.h> #include <library/cpp/threading/poor_man_openmp/thread_helper.h> - -#include <util/system/hp_timer.h> + +#include <util/system/hp_timer.h> #include <util/system/info.h> -#include <util/stream/output.h> -#include <util/datetime/base.h> -#include <util/random/random.h> -#include <util/string/cast.h> -#include <util/generic/xrange.h> -#include <util/generic/algorithm.h> -#include <util/generic/singleton.h> -#include <util/system/spinlock.h> -#include <util/generic/function.h> -#include <util/generic/maybe.h> -#include <util/generic/strbuf.h> -#include <util/generic/intrlist.h> +#include <util/stream/output.h> +#include <util/datetime/base.h> +#include <util/random/random.h> +#include <util/string/cast.h> +#include <util/generic/xrange.h> +#include <util/generic/algorithm.h> +#include <util/generic/singleton.h> +#include <util/system/spinlock.h> +#include <util/generic/function.h> +#include <util/generic/maybe.h> +#include <util/generic/strbuf.h> +#include <util/generic/intrlist.h> #include <util/stream/format.h> -#include <util/system/yield.h> - +#include <util/system/yield.h> + using re2::RE2; -using namespace NBench; -using namespace NColorizer; -using namespace NLastGetopt; - -namespace { - struct TOptions { - double TimeBudget; - }; - - struct TResult { - TStringBuf TestName; - ui64 Samples; - ui64 Iterations; - TMaybe<double> CyclesPerIteration; - TMaybe<double> SecondsPerIteration; - double RunTime; +using namespace NBench; +using namespace NColorizer; +using namespace NLastGetopt; + +namespace { + struct TOptions { + double TimeBudget; + }; + + struct TResult { + TStringBuf TestName; + ui64 Samples; + ui64 Iterations; + TMaybe<double> CyclesPerIteration; + TMaybe<double> SecondsPerIteration; + double RunTime; size_t TestId; // Sequential test id (zero-based) - }; - - struct ITestRunner: public TIntrusiveListItem<ITestRunner> { + }; + + struct ITestRunner: public TIntrusiveListItem<ITestRunner> { virtual ~ITestRunner() = default; - void Register(); - - virtual TStringBuf Name() const noexcept = 0; - virtual TResult Run(const TOptions& opts) = 0; + void Register(); + + virtual TStringBuf Name() const noexcept = 0; + virtual TResult Run(const TOptions& opts) = 0; size_t SequentialId = 0; - }; - - struct TCpuBenchmark: public ITestRunner { - inline TCpuBenchmark(const char* name, NCpu::TUserFunc func) - : F(func) - , N(name) - { - Register(); - } - + }; + + struct TCpuBenchmark: public ITestRunner { + inline TCpuBenchmark(const char* name, NCpu::TUserFunc func) + : F(func) + , N(name) + { + Register(); + } + TResult Run(const TOptions& opts) override; - + TStringBuf Name() const noexcept override { - return N; - } - - std::function<NCpu::TUserFunc> F; - const TStringBuf N; - }; - + return N; + } + + std::function<NCpu::TUserFunc> F; + const TStringBuf N; + }; + inline TString DoFmtTime(double t) { - if (t > 0.1) { - return ToString(t) + " seconds"; - } - - t *= 1000.0; - - if (t > 0.1) { - return ToString(t) + " milliseconds"; - } - - t *= 1000.0; - - if (t > 0.1) { - return ToString(t) + " microseconds"; - } - - t *= 1000.0; - - if (t < 0.05) { - t = 0.0; - } - - return ToString(t) + " nanoseconds"; - } - - struct THiPerfTimer: public THPTimer { + if (t > 0.1) { + return ToString(t) + " seconds"; + } + + t *= 1000.0; + + if (t > 0.1) { + return ToString(t) + " milliseconds"; + } + + t *= 1000.0; + + if (t > 0.1) { + return ToString(t) + " microseconds"; + } + + t *= 1000.0; + + if (t < 0.05) { + t = 0.0; + } + + return ToString(t) + " nanoseconds"; + } + + struct THiPerfTimer: public THPTimer { static inline TString FmtTime(double t) { - return DoFmtTime(t); - } - }; - - struct TSimpleTimer { - inline double Passed() const noexcept { - return (TInstant::Now() - N).MicroSeconds() / 1000000.0; - } - + return DoFmtTime(t); + } + }; + + struct TSimpleTimer { + inline double Passed() const noexcept { + return (TInstant::Now() - N).MicroSeconds() / 1000000.0; + } + static inline TString FmtTime(double t) { - return DoFmtTime(t); - } - - const TInstant N = TInstant::Now(); - }; - - struct TCycleTimer { - inline ui64 Passed() const noexcept { - return GetCycleCount() - N; - } - + return DoFmtTime(t); + } + + const TInstant N = TInstant::Now(); + }; + + struct TCycleTimer { + inline ui64 Passed() const noexcept { + return GetCycleCount() - N; + } + static inline TString FmtTime(double t) { - if (t < 0.5) { - t = 0.0; - } - + if (t < 0.5) { + t = 0.0; + } + TString hr; if (t > 10 * 1000) { hr = " (" + ToString(HumanReadableSize(t, ESizeFormat::SF_QUANTITY)) + ")"; } return ToString(t) + hr + " cycles"; - } - - const ui64 N = GetCycleCount(); - }; - - template <class TMyTimer, class T> - inline double Measure(T&& t, size_t n) { - TMyTimer timer; - - t(n); - - return timer.Passed(); - } - - struct TSampleIterator { - inline size_t Next() noexcept { - return M++; - - N *= 1.02; - M += 1; - - return Max<double>(N, M); - } - - double N = 1.0; - size_t M = 1; - }; - - using TSample = std::pair<size_t, double>; + } + + const ui64 N = GetCycleCount(); + }; + + template <class TMyTimer, class T> + inline double Measure(T&& t, size_t n) { + TMyTimer timer; + + t(n); + + return timer.Passed(); + } + + struct TSampleIterator { + inline size_t Next() noexcept { + return M++; + + N *= 1.02; + M += 1; + + return Max<double>(N, M); + } + + double N = 1.0; + size_t M = 1; + }; + + using TSample = std::pair<size_t, double>; using TSamples = TVector<TSample>; - - struct TLinFunc { - double A; - double B; - - inline double operator()(double x) const noexcept { - return A * x + B; - } - }; - + + struct TLinFunc { + double A; + double B; + + inline double operator()(double x) const noexcept { + return A * x + B; + } + }; + TLinFunc CalcModel(const TSamples& s) { - TKahanSLRSolver solver; - - for (const auto& p : s) { - solver.Add(p.first, p.second); - } - - double c = 0; - double i = 0; - - solver.Solve(c, i); - - return TLinFunc{c, i}; - } - + TKahanSLRSolver solver; + + for (const auto& p : s) { + solver.Add(p.first, p.second); + } + + double c = 0; + double i = 0; + + solver.Solve(c, i); + + return TLinFunc{c, i}; + } + inline TSamples RemoveOutliers(const TSamples& s, double fraction) { if (s.size() < 20) { - return s; - } - - const auto predictor = CalcModel(s); - - const auto errfunc = [&predictor](const TSample& p) -> double { - //return (1.0 + fabs(predictor(p.first) - p.second)) / (1.0 + fabs(p.second)); - //return fabs((predictor(p.first) - p.second)) / (1.0 + fabs(p.second)); - //return fabs((predictor(p.first) - p.second)) / (1.0 + p.first); - return fabs((predictor(p.first) - p.second)); - }; - - using TSampleWithError = std::pair<const TSample*, double>; + return s; + } + + const auto predictor = CalcModel(s); + + const auto errfunc = [&predictor](const TSample& p) -> double { + //return (1.0 + fabs(predictor(p.first) - p.second)) / (1.0 + fabs(p.second)); + //return fabs((predictor(p.first) - p.second)) / (1.0 + fabs(p.second)); + //return fabs((predictor(p.first) - p.second)) / (1.0 + p.first); + return fabs((predictor(p.first) - p.second)); + }; + + using TSampleWithError = std::pair<const TSample*, double>; TVector<TSampleWithError> v; - + v.reserve(s.size()); - - for (const auto& p : s) { - v.emplace_back(&p, errfunc(p)); - } - - Sort(v.begin(), v.end(), [](const TSampleWithError& l, const TSampleWithError& r) -> bool { - return (l.second < r.second) || ((l.second == r.second) && (l.first < r.first)); - }); - - if (0) { - for (const auto& x : v) { - Cout << x.first->first << ", " << x.first->second << " -> " << x.second << Endl; - } - } - - TSamples ret; - + + for (const auto& p : s) { + v.emplace_back(&p, errfunc(p)); + } + + Sort(v.begin(), v.end(), [](const TSampleWithError& l, const TSampleWithError& r) -> bool { + return (l.second < r.second) || ((l.second == r.second) && (l.first < r.first)); + }); + + if (0) { + for (const auto& x : v) { + Cout << x.first->first << ", " << x.first->second << " -> " << x.second << Endl; + } + } + + TSamples ret; + ret.reserve(v.size()); - + for (const auto i : xrange<size_t>(0, fraction * v.size())) { - ret.push_back(*v[i].first); - } - - return ret; - } - - template <class TMyTimer, class T> - static inline TResult RunTest(T&& func, double budget, ITestRunner& test) { - THPTimer start; - - start.Passed(); - - TSampleIterator sample; - TSamples samples; - ui64 iters = 0; - - //warm up - func(1); - - while (start.Passed() < budget) { + ret.push_back(*v[i].first); + } + + return ret; + } + + template <class TMyTimer, class T> + static inline TResult RunTest(T&& func, double budget, ITestRunner& test) { + THPTimer start; + + start.Passed(); + + TSampleIterator sample; + TSamples samples; + ui64 iters = 0; + + //warm up + func(1); + + while (start.Passed() < budget) { if (start.Passed() < ((budget * samples.size()) / 2000000.0)) { - ThreadYield(); - } else { - const size_t n = sample.Next(); - - iters += (ui64)n; - samples.emplace_back(n, Measure<TMyTimer>(func, n)); - } - } - - auto filtered = RemoveOutliers(samples, 0.9); - + ThreadYield(); + } else { + const size_t n = sample.Next(); + + iters += (ui64)n; + samples.emplace_back(n, Measure<TMyTimer>(func, n)); + } + } + + auto filtered = RemoveOutliers(samples, 0.9); + return {test.Name(), filtered.size(), iters, CalcModel(filtered).A, Nothing(), start.Passed(), test.SequentialId}; - } - - using TTests = TIntrusiveListWithAutoDelete<ITestRunner, TDestructor>; - + } + + using TTests = TIntrusiveListWithAutoDelete<ITestRunner, TDestructor>; + inline TTests& Tests() { - return *Singleton<TTests>(); - } - - void ITestRunner::Register() { - Tests().PushBack(this); - } - - TResult TCpuBenchmark::Run(const TOptions& opts) { - return RunTest<TCycleTimer>([this](size_t n) { - NCpu::TParams params{n}; - - F(params); + return *Singleton<TTests>(); + } + + void ITestRunner::Register() { + Tests().PushBack(this); + } + + TResult TCpuBenchmark::Run(const TOptions& opts) { + return RunTest<TCycleTimer>([this](size_t n) { + NCpu::TParams params{n}; + + F(params); }, opts.TimeBudget, *this); - } + } enum EOutFormat { F_CONSOLE = 0 /* "console" */, - F_CSV /* "csv" */, - F_JSON /* "json" */ + F_CSV /* "csv" */, + F_JSON /* "json" */ }; TAdaptiveLock STDOUT_LOCK; - struct IReporter { + struct IReporter { virtual void Report(TResult&& result) = 0; virtual void Finish() { @@ -297,7 +297,7 @@ namespace { } }; - class TConsoleReporter: public IReporter { + class TConsoleReporter: public IReporter { public: ~TConsoleReporter() override { } @@ -309,7 +309,7 @@ namespace { } }; - class TCSVReporter: public IReporter { + class TCSVReporter: public IReporter { public: TCSVReporter() { Cout << "Name\tSamples\tIterations\tRun_time\tPer_iteration_sec\tPer_iteration_cycles" << Endl; @@ -344,7 +344,7 @@ namespace { } }; - class TJSONReporter: public IReporter { + class TJSONReporter: public IReporter { public: ~TJSONReporter() override { } @@ -427,15 +427,15 @@ namespace { case F_CSV: return MakeHolder<TCSVReporter>(); - + case F_JSON: return MakeHolder<TJSONReporter>(); - - default: - break; + + default: + break; } - - return MakeHolder<TConsoleReporter>(); // make compiler happy + + return MakeHolder<TConsoleReporter>(); // make compiler happy } THolder<IReporter> MakeOrderedReporter(const EOutFormat type) { @@ -447,12 +447,12 @@ namespace { tests[id]->SequentialId = id; } } -} +} template <> EOutFormat FromStringImpl<EOutFormat>(const char* data, size_t len) { const auto s = TStringBuf{data, len}; - + if (TStringBuf("console") == s) { return F_CONSOLE; } else if (TStringBuf("csv") == s) { @@ -462,55 +462,55 @@ EOutFormat FromStringImpl<EOutFormat>(const char* data, size_t len) { } ythrow TFromStringException{} << "failed to convert '" << s << '\''; -} - -template <> +} + +template <> void Out<TResult>(IOutputStream& out, const TResult& r) { - out << "----------- " << LightRed() << r.TestName << Old() << " ---------------" << Endl - << " samples: " << White() << r.Samples << Old() << Endl - << " iterations: " << White() << r.Iterations << Old() << Endl + out << "----------- " << LightRed() << r.TestName << Old() << " ---------------" << Endl + << " samples: " << White() << r.Samples << Old() << Endl + << " iterations: " << White() << r.Iterations << Old() << Endl << " iterations hr: " << White() << HumanReadableSize(r.Iterations, SF_QUANTITY) << Old() << Endl - << " run time: " << White() << r.RunTime << Old() << Endl; - - if (r.CyclesPerIteration) { - out << " per iteration: " << White() << TCycleTimer::FmtTime(*r.CyclesPerIteration) << Old() << Endl; - } - - if (r.SecondsPerIteration) { - out << " per iteration: " << White() << DoFmtTime(*r.SecondsPerIteration) << Old() << Endl; - } -} - -NCpu::TRegistar::TRegistar(const char* name, TUserFunc func) { + << " run time: " << White() << r.RunTime << Old() << Endl; + + if (r.CyclesPerIteration) { + out << " per iteration: " << White() << TCycleTimer::FmtTime(*r.CyclesPerIteration) << Old() << Endl; + } + + if (r.SecondsPerIteration) { + out << " per iteration: " << White() << DoFmtTime(*r.SecondsPerIteration) << Old() << Endl; + } +} + +NCpu::TRegistar::TRegistar(const char* name, TUserFunc func) { static_assert(sizeof(TCpuBenchmark) + alignof(TCpuBenchmark) < sizeof(Buf), "fix Buf size"); - + new (AlignUp(Buf, alignof(TCpuBenchmark))) TCpuBenchmark(name, func); -} - -namespace { - struct TProgOpts { - TProgOpts(int argc, char** argv) { - TOpts opts = TOpts::Default(); - - opts.AddHelpOption(); - - opts.AddLongOption('b', "budget") - .StoreResult(&TimeBudget) +} + +namespace { + struct TProgOpts { + TProgOpts(int argc, char** argv) { + TOpts opts = TOpts::Default(); + + opts.AddHelpOption(); + + opts.AddLongOption('b', "budget") + .StoreResult(&TimeBudget) .RequiredArgument("SEC") - .Optional() - .Help("overall time budget"); - - opts.AddLongOption('l', "list") + .Optional() + .Help("overall time budget"); + + opts.AddLongOption('l', "list") .NoArgument() .StoreValue(&ListTests, true) - .Help("list all tests"); - - opts.AddLongOption('t', "threads") - .StoreResult(&Threads) + .Help("list all tests"); + + opts.AddLongOption('t', "threads") + .StoreResult(&Threads) .OptionalValue(ToString((NSystemInfo::CachedNumberOfCpus() + 1) / 2), "JOBS") .DefaultValue("1") - .Help("run benchmarks in parallel"); - + .Help("run benchmarks in parallel"); + opts.AddLongOption('f', "format") .AddLongName("benchmark_format") .StoreResult(&OutFormat) @@ -521,18 +521,18 @@ namespace { opts.SetFreeArgDefaultTitle("REGEXP", "RE2 regular expression to filter tests"); const TOptsParseResult parseResult{&opts, argc, argv}; - + for (const auto& regexp : parseResult.GetFreeArgs()) { Filters.push_back(MakeHolder<RE2>(regexp.data(), RE2::Quiet)); Y_ENSURE(Filters.back()->ok(), "incorrect RE2 expression '" << regexp << "'"); } - } - - bool MatchFilters(const TStringBuf& name) const { - if (!Filters) { - return true; - } - + } + + bool MatchFilters(const TStringBuf& name) const { + if (!Filters) { + return true; + } + for (auto&& re : Filters) { if (RE2::FullMatchN({name.data(), name.size()}, *re, nullptr, 0)) { return true; @@ -542,50 +542,50 @@ namespace { return false; } - bool ListTests = false; - double TimeBudget = -1.0; + bool ListTests = false; + double TimeBudget = -1.0; TVector<THolder<RE2>> Filters; - size_t Threads = 0; + size_t Threads = 0; EOutFormat OutFormat; - }; -} - + }; +} + int NBench::Main(int argc, char** argv) { const TProgOpts opts(argc, argv); - + TVector<ITestRunner*> tests; for (auto&& it : Tests()) { if (opts.MatchFilters(it.Name())) { tests.push_back(&it); - } + } } EnumerateTests(tests); - + if (opts.ListTests) { for (const auto* const it : tests) { Cout << it->Name() << Endl; - } - + } + return 0; } - + if (!tests) { return 0; } - + double timeBudget = opts.TimeBudget; - + if (timeBudget < 0) { timeBudget = 5.0 * tests.size(); } - + const TOptions testOpts = {timeBudget / tests.size()}; const auto reporter = MakeOrderedReporter(opts.OutFormat); - - std::function<void(ITestRunner**)> func = [&](ITestRunner** it) { + + std::function<void(ITestRunner**)> func = [&](ITestRunner** it) { auto&& res = (*it)->Run(testOpts); - + reporter->Report(std::move(res)); }; @@ -596,9 +596,9 @@ int NBench::Main(int argc, char** argv) { for (auto it : tests) { func(&it); } - } + } reporter->Finish(); return 0; -} +} diff --git a/library/cpp/testing/benchmark/bench.h b/library/cpp/testing/benchmark/bench.h index 21551ad0dd..6f7a9f3f44 100644 --- a/library/cpp/testing/benchmark/bench.h +++ b/library/cpp/testing/benchmark/bench.h @@ -1,29 +1,29 @@ -#pragma once - +#pragma once + #include <util/system/compiler.h> -#include <util/system/types.h> - +#include <util/system/types.h> + #include <utility> -namespace NBench { - namespace NCpu { - struct TParams { - inline size_t Iterations() const noexcept { - return Iterations_; - } - - const size_t Iterations_; - }; - - using TUserFunc = void(TParams&); - - struct TRegistar { - TRegistar(const char* name, TUserFunc func); - - char Buf[128]; - }; - } - +namespace NBench { + namespace NCpu { + struct TParams { + inline size_t Iterations() const noexcept { + return Iterations_; + } + + const size_t Iterations_; + }; + + using TUserFunc = void(TParams&); + + struct TRegistar { + TRegistar(const char* name, TUserFunc func); + + char Buf[128]; + }; + } + /** * Functions that states "I can read and write everywhere in memory". * @@ -42,10 +42,10 @@ namespace NBench { #if defined(__GNUC__) Y_FORCE_INLINE void Clobber() { - asm volatile("" - : - : - : "memory"); + asm volatile("" + : + : + : "memory"); } #elif defined(_MSC_VER) Y_FORCE_INLINE void Clobber() { @@ -60,10 +60,10 @@ namespace NBench { #if defined(__GNUC__) template <typename T> Y_FORCE_INLINE void Escape(T* p) { - asm volatile("" - : - : "g"(p) - : "memory"); + asm volatile("" + : + : "g"(p) + : "memory"); } #else template <typename T> @@ -77,16 +77,16 @@ namespace NBench { * @param Unused variable (e.g. return value of benchmarked function). */ template <typename T> - Y_FORCE_INLINE void DoNotOptimize(T&& datum) { + Y_FORCE_INLINE void DoNotOptimize(T&& datum) { ::DoNotOptimizeAway(std::forward<T>(datum)); - } + } int Main(int argc, char** argv); -} - +} + #define Y_CPU_BENCHMARK(name, cnt) \ namespace N_bench_##name { \ static void Run(::NBench::NCpu::TParams&); \ const ::NBench::NCpu::TRegistar benchmark(#name, &Run); \ } \ - static void N_bench_##name::Run(::NBench::NCpu::TParams& cnt) + static void N_bench_##name::Run(::NBench::NCpu::TParams& cnt) diff --git a/library/cpp/testing/benchmark/examples/main.cpp b/library/cpp/testing/benchmark/examples/main.cpp index ddd8b05ffc..508d12111e 100644 --- a/library/cpp/testing/benchmark/examples/main.cpp +++ b/library/cpp/testing/benchmark/examples/main.cpp @@ -1,186 +1,186 @@ #include <library/cpp/testing/benchmark/bench.h> - -#include <util/generic/xrange.h> -#include <util/generic/algorithm.h> -#include <util/generic/vector.h> -#include <util/generic/yexception.h> -#include <util/generic/bt_exception.h> - -Y_CPU_BENCHMARK(F, iface) { + +#include <util/generic/xrange.h> +#include <util/generic/algorithm.h> +#include <util/generic/vector.h> +#include <util/generic/yexception.h> +#include <util/generic/bt_exception.h> + +Y_CPU_BENCHMARK(F, iface) { TVector<size_t> x; - - x.reserve(iface.Iterations()); - - for (size_t i = 0; i < iface.Iterations(); ++i) { - x.push_back(i); - } -} - -Y_CPU_BENCHMARK(EmptyF, iface) { - (void)iface; -} - -Y_CPU_BENCHMARK(AlmostEmptyF, iface) { - (void)iface; - + + x.reserve(iface.Iterations()); + + for (size_t i = 0; i < iface.Iterations(); ++i) { + x.push_back(i); + } +} + +Y_CPU_BENCHMARK(EmptyF, iface) { + (void)iface; +} + +Y_CPU_BENCHMARK(AlmostEmptyF, iface) { + (void)iface; + TVector<size_t> x; - x.resize(1); -} - -Y_CPU_BENCHMARK(TestThrow, iface) { - for (size_t i = 0; i < iface.Iterations(); ++i) { - try { - ythrow yexception() << i; - } catch (...) { - //CurrentExceptionMessage(); - } - } -} - -Y_CPU_BENCHMARK(TestThrowBT, iface) { - for (size_t i = 0; i < iface.Iterations(); ++i) { - try { - ythrow TWithBackTrace<yexception>() << i; - } catch (...) { - //CurrentExceptionMessage(); - } - } -} - -Y_CPU_BENCHMARK(TestThrowCatch, iface) { - for (size_t i = 0; i < iface.Iterations(); ++i) { - try { - ythrow yexception() << i; - } catch (...) { - Y_DO_NOT_OPTIMIZE_AWAY(CurrentExceptionMessage()); - } - } -} - -Y_CPU_BENCHMARK(TestThrowCatchBT, iface) { - for (size_t i = 0; i < iface.Iterations(); ++i) { - try { - ythrow TWithBackTrace<yexception>() << i; - } catch (...) { - Y_DO_NOT_OPTIMIZE_AWAY(CurrentExceptionMessage()); - } - } -} - -Y_CPU_BENCHMARK(TestRobust, iface) { - if (iface.Iterations() % 100 == 0) { - usleep(100000); - } -} - -Y_CPU_BENCHMARK(IterationSpeed, iface) { - const auto n = iface.Iterations(); - - for (size_t i = 0; i < n; ++i) { - Y_DO_NOT_OPTIMIZE_AWAY(i); - } -} - -Y_CPU_BENCHMARK(XRangeSpeed, iface) { - for (auto i : xrange<size_t>(0, iface.Iterations())) { - Y_DO_NOT_OPTIMIZE_AWAY(i); - } -} - -Y_NO_INLINE int FFF() { - return 0; -} - -Y_NO_INLINE int FFF(int x) { - return x; -} - -Y_NO_INLINE int FFF(int x, int y) { - return x + y; -} - -Y_NO_INLINE size_t FS1(TStringBuf x) { + x.resize(1); +} + +Y_CPU_BENCHMARK(TestThrow, iface) { + for (size_t i = 0; i < iface.Iterations(); ++i) { + try { + ythrow yexception() << i; + } catch (...) { + //CurrentExceptionMessage(); + } + } +} + +Y_CPU_BENCHMARK(TestThrowBT, iface) { + for (size_t i = 0; i < iface.Iterations(); ++i) { + try { + ythrow TWithBackTrace<yexception>() << i; + } catch (...) { + //CurrentExceptionMessage(); + } + } +} + +Y_CPU_BENCHMARK(TestThrowCatch, iface) { + for (size_t i = 0; i < iface.Iterations(); ++i) { + try { + ythrow yexception() << i; + } catch (...) { + Y_DO_NOT_OPTIMIZE_AWAY(CurrentExceptionMessage()); + } + } +} + +Y_CPU_BENCHMARK(TestThrowCatchBT, iface) { + for (size_t i = 0; i < iface.Iterations(); ++i) { + try { + ythrow TWithBackTrace<yexception>() << i; + } catch (...) { + Y_DO_NOT_OPTIMIZE_AWAY(CurrentExceptionMessage()); + } + } +} + +Y_CPU_BENCHMARK(TestRobust, iface) { + if (iface.Iterations() % 100 == 0) { + usleep(100000); + } +} + +Y_CPU_BENCHMARK(IterationSpeed, iface) { + const auto n = iface.Iterations(); + + for (size_t i = 0; i < n; ++i) { + Y_DO_NOT_OPTIMIZE_AWAY(i); + } +} + +Y_CPU_BENCHMARK(XRangeSpeed, iface) { + for (auto i : xrange<size_t>(0, iface.Iterations())) { + Y_DO_NOT_OPTIMIZE_AWAY(i); + } +} + +Y_NO_INLINE int FFF() { + return 0; +} + +Y_NO_INLINE int FFF(int x) { + return x; +} + +Y_NO_INLINE int FFF(int x, int y) { + return x + y; +} + +Y_NO_INLINE size_t FS1(TStringBuf x) { return x.size(); -} - -Y_NO_INLINE size_t FS1_2(TStringBuf x, TStringBuf y) { +} + +Y_NO_INLINE size_t FS1_2(TStringBuf x, TStringBuf y) { return x.size() + y.size(); -} - -Y_NO_INLINE size_t FS2(const TStringBuf& x) { +} + +Y_NO_INLINE size_t FS2(const TStringBuf& x) { return x.size(); -} - -Y_NO_INLINE size_t FS2_2(const TStringBuf& x, const TStringBuf& y) { +} + +Y_NO_INLINE size_t FS2_2(const TStringBuf& x, const TStringBuf& y) { return x.size() + y.size(); -} - -Y_CPU_BENCHMARK(FunctionCallCost_StringBufVal1, iface) { - TStringBuf x; - - for (auto i : xrange<size_t>(0, iface.Iterations())) { - (void)i; +} + +Y_CPU_BENCHMARK(FunctionCallCost_StringBufVal1, iface) { + TStringBuf x; + + for (auto i : xrange<size_t>(0, iface.Iterations())) { + (void)i; NBench::Escape(&x); - Y_DO_NOT_OPTIMIZE_AWAY(FS1(x)); + Y_DO_NOT_OPTIMIZE_AWAY(FS1(x)); NBench::Clobber(); - } -} - -Y_CPU_BENCHMARK(FunctionCallCost_StringBufRef1, iface) { - TStringBuf x; - - for (auto i : xrange<size_t>(0, iface.Iterations())) { - (void)i; + } +} + +Y_CPU_BENCHMARK(FunctionCallCost_StringBufRef1, iface) { + TStringBuf x; + + for (auto i : xrange<size_t>(0, iface.Iterations())) { + (void)i; NBench::Escape(&x); - Y_DO_NOT_OPTIMIZE_AWAY(FS2(x)); + Y_DO_NOT_OPTIMIZE_AWAY(FS2(x)); NBench::Clobber(); - } -} - -Y_CPU_BENCHMARK(FunctionCallCost_StringBufVal2, iface) { - TStringBuf x; - TStringBuf y; - - for (auto i : xrange<size_t>(0, iface.Iterations())) { - (void)i; + } +} + +Y_CPU_BENCHMARK(FunctionCallCost_StringBufVal2, iface) { + TStringBuf x; + TStringBuf y; + + for (auto i : xrange<size_t>(0, iface.Iterations())) { + (void)i; NBench::Escape(&x); NBench::Escape(&y); - Y_DO_NOT_OPTIMIZE_AWAY(FS1_2(x, y)); + Y_DO_NOT_OPTIMIZE_AWAY(FS1_2(x, y)); NBench::Clobber(); - } -} - -Y_CPU_BENCHMARK(FunctionCallCost_StringBufRef2, iface) { - TStringBuf x; - TStringBuf y; - - for (auto i : xrange<size_t>(0, iface.Iterations())) { - (void)i; + } +} + +Y_CPU_BENCHMARK(FunctionCallCost_StringBufRef2, iface) { + TStringBuf x; + TStringBuf y; + + for (auto i : xrange<size_t>(0, iface.Iterations())) { + (void)i; NBench::Escape(&x); NBench::Escape(&y); - Y_DO_NOT_OPTIMIZE_AWAY(FS2_2(x, y)); + Y_DO_NOT_OPTIMIZE_AWAY(FS2_2(x, y)); NBench::Clobber(); - } -} - -Y_CPU_BENCHMARK(FunctionCallCost_NoArg, iface) { - for (auto i : xrange<size_t>(0, iface.Iterations())) { - (void)i; - Y_DO_NOT_OPTIMIZE_AWAY(FFF()); - } -} - -Y_CPU_BENCHMARK(FunctionCallCost_OneArg, iface) { - for (auto i : xrange<size_t>(0, iface.Iterations())) { - Y_DO_NOT_OPTIMIZE_AWAY(FFF(i)); - } -} - -Y_CPU_BENCHMARK(FunctionCallCost_TwoArg, iface) { - for (auto i : xrange<size_t>(0, iface.Iterations())) { - Y_DO_NOT_OPTIMIZE_AWAY(FFF(i, i)); - } -} + } +} + +Y_CPU_BENCHMARK(FunctionCallCost_NoArg, iface) { + for (auto i : xrange<size_t>(0, iface.Iterations())) { + (void)i; + Y_DO_NOT_OPTIMIZE_AWAY(FFF()); + } +} + +Y_CPU_BENCHMARK(FunctionCallCost_OneArg, iface) { + for (auto i : xrange<size_t>(0, iface.Iterations())) { + Y_DO_NOT_OPTIMIZE_AWAY(FFF(i)); + } +} + +Y_CPU_BENCHMARK(FunctionCallCost_TwoArg, iface) { + for (auto i : xrange<size_t>(0, iface.Iterations())) { + Y_DO_NOT_OPTIMIZE_AWAY(FFF(i, i)); + } +} /* An example of incorrect benchmark. As of r2581591 Clang 3.7 produced following assembly: * @code diff --git a/library/cpp/testing/benchmark/examples/ya.make b/library/cpp/testing/benchmark/examples/ya.make index 7e696e127a..c5c435ce10 100644 --- a/library/cpp/testing/benchmark/examples/ya.make +++ b/library/cpp/testing/benchmark/examples/ya.make @@ -4,9 +4,9 @@ OWNER( ) Y_BENCHMARK() - -SRCS( - main.cpp -) - -END() + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/testing/benchmark/main/main.cpp b/library/cpp/testing/benchmark/main/main.cpp index aabcb89c43..3ef4a4f348 100644 --- a/library/cpp/testing/benchmark/main/main.cpp +++ b/library/cpp/testing/benchmark/main/main.cpp @@ -8,9 +8,9 @@ int main(int argc, char** argv) { try { return NBench::Main(argc, argv); - } catch (...) { + } catch (...) { Cerr << CurrentExceptionMessage() << Endl; } - - return EXIT_FAILURE; + + return EXIT_FAILURE; } diff --git a/library/cpp/testing/benchmark/main/ya.make b/library/cpp/testing/benchmark/main/ya.make index d00cdcf9fc..9a088cfcea 100644 --- a/library/cpp/testing/benchmark/main/ya.make +++ b/library/cpp/testing/benchmark/main/ya.make @@ -6,7 +6,7 @@ OWNER( ) SRCS( - GLOBAL main.cpp + GLOBAL main.cpp ) PEERDIR( diff --git a/library/cpp/testing/benchmark/ya.make b/library/cpp/testing/benchmark/ya.make index f42be80698..a2c23a50fb 100644 --- a/library/cpp/testing/benchmark/ya.make +++ b/library/cpp/testing/benchmark/ya.make @@ -1,12 +1,12 @@ -LIBRARY() - +LIBRARY() + OWNER( pg yazevnul ) SRCS( - bench.cpp + bench.cpp dummy.cpp ) diff --git a/library/cpp/testing/gmock_in_unittest/gmock.h b/library/cpp/testing/gmock_in_unittest/gmock.h index 31f6aee1c3..5406c58e6e 100644 --- a/library/cpp/testing/gmock_in_unittest/gmock.h +++ b/library/cpp/testing/gmock_in_unittest/gmock.h @@ -1,5 +1,5 @@ #pragma once - + #include <library/cpp/testing/gtest_extensions/gtest_extensions.h> #include <gmock/gmock.h> diff --git a/library/cpp/testing/gmock_in_unittest/registration.cpp b/library/cpp/testing/gmock_in_unittest/registration.cpp index c2872a4c27..e6186a1bc6 100644 --- a/library/cpp/testing/gmock_in_unittest/registration.cpp +++ b/library/cpp/testing/gmock_in_unittest/registration.cpp @@ -17,4 +17,4 @@ namespace { NUnitTest::NPlugin::TPluginRegistrator registerGMock(new TGMockUnittestPlugin()); -} +} diff --git a/library/cpp/testing/unittest/checks.cpp b/library/cpp/testing/unittest/checks.cpp index c5712ae9d2..95baf5c9c2 100644 --- a/library/cpp/testing/unittest/checks.cpp +++ b/library/cpp/testing/unittest/checks.cpp @@ -6,7 +6,7 @@ bool CheckExceptionMessage(const char* msg, TString& err) { // Операция успешно завершена [cp1251] "\xce\xef\xe5\xf0\xe0\xf6\xe8\xff\x20\xf3\xf1\xef\xe5\xf8\xed\xee\x20\xe7\xe0\xe2\xe5\xf0\xf8\xe5\xed\xe0", "The operation completed successfully", - "No error"}; + "No error"}; err.clear(); diff --git a/library/cpp/testing/unittest/env.h b/library/cpp/testing/unittest/env.h index 4807539ab2..568a1c8e8f 100644 --- a/library/cpp/testing/unittest/env.h +++ b/library/cpp/testing/unittest/env.h @@ -1,3 +1,3 @@ // just shortcut #include <library/cpp/testing/common/env.h> - + diff --git a/library/cpp/testing/unittest/example_ut.cpp b/library/cpp/testing/unittest/example_ut.cpp index bcc1ce33f0..52790f1fc6 100644 --- a/library/cpp/testing/unittest/example_ut.cpp +++ b/library/cpp/testing/unittest/example_ut.cpp @@ -1,12 +1,12 @@ #include <library/cpp/testing/unittest/registar.h> - -/* - * just copy-paste it for good start point - */ - + +/* + * just copy-paste it for good start point + */ + Y_UNIT_TEST_SUITE(TUnitTest) { Y_UNIT_TEST(TestEqual) { - UNIT_ASSERT_EQUAL(0, 0); - UNIT_ASSERT_EQUAL(1, 1); - } -} + UNIT_ASSERT_EQUAL(0, 0); + UNIT_ASSERT_EQUAL(1, 1); + } +} diff --git a/library/cpp/testing/unittest/gtest.cpp b/library/cpp/testing/unittest/gtest.cpp index ebad1ea4d6..38f29dcdf8 100644 --- a/library/cpp/testing/unittest/gtest.cpp +++ b/library/cpp/testing/unittest/gtest.cpp @@ -1,67 +1,67 @@ -#include "gtest.h" +#include "gtest.h" #include "simple.h" - -#include <util/generic/map.h> -#include <util/generic/vector.h> + +#include <util/generic/map.h> +#include <util/generic/vector.h> #include <util/system/type_name.h> - -using namespace NUnitTest; -using namespace NUnitTest::NPrivate; - + +using namespace NUnitTest; +using namespace NUnitTest::NPrivate; + IGTestFactory::~IGTestFactory() { -} - -namespace { +} + +namespace { struct TCurrentTest: public TSimpleTestExecutor { - inline TCurrentTest(TStringBuf name) - : MyName(name) - { - } - + inline TCurrentTest(TStringBuf name) + : MyName(name) + { + } + TString TypeId() const override { return TypeName(*this) + "-" + MyName; - } - + } + TString Name() const noexcept override { return TString(MyName); - } - - const TStringBuf MyName; - }; - - struct TGTestFactory: public IGTestFactory { - inline TGTestFactory(TStringBuf name) - : Test(name) - { - } - + } + + const TStringBuf MyName; + }; + + struct TGTestFactory: public IGTestFactory { + inline TGTestFactory(TStringBuf name) + : Test(name) + { + } + ~TGTestFactory() override { - } - + } + TString Name() const noexcept override { - return Test.Name(); - } - + return Test.Name(); + } + TTestBase* ConstructTest() override { - return new TCurrentTest(Test); - } - + return new TCurrentTest(Test); + } + void AddTest(const char* name, void (*body)(TTestContext&), bool forceFork) override { Test.Tests.push_back(TBaseTestCase(name, body, forceFork)); - } - - TCurrentTest Test; - }; -} - -IGTestFactory* NUnitTest::NPrivate::ByName(const char* name) { + } + + TCurrentTest Test; + }; +} + +IGTestFactory* NUnitTest::NPrivate::ByName(const char* name) { static TMap<TStringBuf, TAutoPtr<TGTestFactory>> tests; - - auto& ret = tests[name]; - - if (!ret) { - ret = new TGTestFactory(name); - } - - return ret.Get(); -} + + auto& ret = tests[name]; + + if (!ret) { + ret = new TGTestFactory(name); + } + + return ret.Get(); +} diff --git a/library/cpp/testing/unittest/gtest.h b/library/cpp/testing/unittest/gtest.h index b6768b1bf0..9db9e89757 100644 --- a/library/cpp/testing/unittest/gtest.h +++ b/library/cpp/testing/unittest/gtest.h @@ -1,108 +1,108 @@ -#pragma once - +#pragma once + // WARNING: this is a legacy header that tries to mimic the gtest interface while using unittest // under the hood. Avoid using this interface -- use the genuine gtest instead (the GTEST macro). // If you're already using GTEST macro and you've found yourself here, you probably meant // to include `library/cpp/testing/gtest/gtest.h`. -#include "registar.h" - -#include <util/generic/ymath.h> -#include <util/generic/ylimits.h> - -namespace NUnitTest { - namespace NPrivate { - struct IGTestFactory: public ITestBaseFactory { +#include "registar.h" + +#include <util/generic/ymath.h> +#include <util/generic/ylimits.h> + +namespace NUnitTest { + namespace NPrivate { + struct IGTestFactory: public ITestBaseFactory { ~IGTestFactory() override; - + virtual void AddTest(const char* name, void (*body)(TTestContext&), bool forceFork) = 0; - }; - - IGTestFactory* ByName(const char* name); - } -} - -namespace NTesting { - struct TTest { - virtual void SetUp() { - } - - virtual void TearDown() { - } - + }; + + IGTestFactory* ByName(const char* name); + } +} + +namespace NTesting { + struct TTest { + virtual void SetUp() { + } + + virtual void TearDown() { + } + inline TTest* _This() noexcept { - return this; - } - }; -} - -namespace testing { - struct Test: public ::NTesting::TTest { - }; -} - -#define TEST_IMPL(N, NN, FF) \ + return this; + } + }; +} + +namespace testing { + struct Test: public ::NTesting::TTest { + }; +} + +#define TEST_IMPL(N, NN, FF) \ void Test##N##NN(NUnitTest::TTestContext&); \ - namespace NTestSuite##N##NN { \ - struct TReg { \ - inline TReg() { \ - ::NUnitTest::NPrivate::ByName(#N)->AddTest(#NN, &(Test##N##NN), FF); \ - } \ - }; \ - static TReg reg; \ - } \ + namespace NTestSuite##N##NN { \ + struct TReg { \ + inline TReg() { \ + ::NUnitTest::NPrivate::ByName(#N)->AddTest(#NN, &(Test##N##NN), FF); \ + } \ + }; \ + static TReg reg; \ + } \ void Test##N##NN(NUnitTest::TTestContext&) - -#define TEST_F_IMPL(N, NN, FF) \ - namespace NTestSuite##N##NN { \ - struct TTestSuite: public N { \ - inline TTestSuite() { \ - this->_This()->SetUp(); \ - } \ - inline ~TTestSuite() { \ - this->_This()->TearDown(); \ - } \ - void NN(); \ - }; \ - }; \ - TEST_IMPL(N, NN, FF) { \ - NTestSuite##N##NN::TTestSuite().NN(); \ - } \ - void NTestSuite##N##NN::TTestSuite::NN() - -#define TEST(A, B) TEST_IMPL(A, B, false) -#define TEST_FORKED(A, B) TEST_IMPL(A, B, true) - -#define TEST_F(A, B) TEST_F_IMPL(A, B, false) -#define TEST_F_FORKED(A, B) TEST_F_IMPL(A, B, true) - -#define EXPECT_EQ(A, B) UNIT_ASSERT_VALUES_EQUAL(A, B) -#define EXPECT_NE(A, B) UNIT_ASSERT_UNEQUAL(A, B) + +#define TEST_F_IMPL(N, NN, FF) \ + namespace NTestSuite##N##NN { \ + struct TTestSuite: public N { \ + inline TTestSuite() { \ + this->_This()->SetUp(); \ + } \ + inline ~TTestSuite() { \ + this->_This()->TearDown(); \ + } \ + void NN(); \ + }; \ + }; \ + TEST_IMPL(N, NN, FF) { \ + NTestSuite##N##NN::TTestSuite().NN(); \ + } \ + void NTestSuite##N##NN::TTestSuite::NN() + +#define TEST(A, B) TEST_IMPL(A, B, false) +#define TEST_FORKED(A, B) TEST_IMPL(A, B, true) + +#define TEST_F(A, B) TEST_F_IMPL(A, B, false) +#define TEST_F_FORKED(A, B) TEST_F_IMPL(A, B, true) + +#define EXPECT_EQ(A, B) UNIT_ASSERT_VALUES_EQUAL(A, B) +#define EXPECT_NE(A, B) UNIT_ASSERT_UNEQUAL(A, B) #define EXPECT_LE(A, B) UNIT_ASSERT((A) <= (B)) -#define EXPECT_LT(A, B) UNIT_ASSERT((A) < (B)) +#define EXPECT_LT(A, B) UNIT_ASSERT((A) < (B)) #define EXPECT_GE(A, B) UNIT_ASSERT((A) >= (B)) -#define EXPECT_GT(A, B) UNIT_ASSERT((A) > (B)) -#define EXPECT_NO_THROW(A) UNIT_ASSERT_NO_EXCEPTION(A) -#define EXPECT_THROW(A, B) UNIT_ASSERT_EXCEPTION(A, B) -#define EXPECT_NEAR(A, B, D) UNIT_ASSERT_DOUBLES_EQUAL(A, B, D) -#define EXPECT_STREQ(A, B) UNIT_ASSERT_VALUES_EQUAL(A, B) - +#define EXPECT_GT(A, B) UNIT_ASSERT((A) > (B)) +#define EXPECT_NO_THROW(A) UNIT_ASSERT_NO_EXCEPTION(A) +#define EXPECT_THROW(A, B) UNIT_ASSERT_EXCEPTION(A, B) +#define EXPECT_NEAR(A, B, D) UNIT_ASSERT_DOUBLES_EQUAL(A, B, D) +#define EXPECT_STREQ(A, B) UNIT_ASSERT_VALUES_EQUAL(A, B) + #define EXPECT_DOUBLE_EQ_TOLERANCE(A, B, tolerance) UNIT_ASSERT_C(fabs((A) - (B)) < tolerance * std::numeric_limits<decltype(A)>::epsilon(), TString("\n") + ToString(A) + " <> " + ToString(B)) #define EXPECT_DOUBLE_EQ(A, B) EXPECT_DOUBLE_EQ_TOLERANCE(A, B, 4.0) - -//conflicts with util/system/defaults.h -#undef EXPECT_TRUE -#define EXPECT_TRUE(X) UNIT_ASSERT(X) -#undef EXPECT_FALSE -#define EXPECT_FALSE(X) UNIT_ASSERT(!(X)) - -#define ASSERT_EQ(A, B) EXPECT_EQ(A, B) -#define ASSERT_NE(A, B) EXPECT_NE(A, B) -#define ASSERT_GT(A, B) EXPECT_GT(A, B) -#define ASSERT_LT(A, B) EXPECT_LT(A, B) -#define ASSERT_FALSE(X) EXPECT_FALSE(X) -#define ASSERT_TRUE(X) EXPECT_TRUE(X) -#define ASSERT_THROW(A, B) EXPECT_THROW(A, B) -#define ASSERT_NO_THROW(A) EXPECT_NO_THROW(A) -#define ASSERT_DOUBLE_EQ(A, B) EXPECT_DOUBLE_EQ(A, B) -#define ASSERT_STREQ(A, B) EXPECT_STREQ(A, B) + +//conflicts with util/system/defaults.h +#undef EXPECT_TRUE +#define EXPECT_TRUE(X) UNIT_ASSERT(X) +#undef EXPECT_FALSE +#define EXPECT_FALSE(X) UNIT_ASSERT(!(X)) + +#define ASSERT_EQ(A, B) EXPECT_EQ(A, B) +#define ASSERT_NE(A, B) EXPECT_NE(A, B) +#define ASSERT_GT(A, B) EXPECT_GT(A, B) +#define ASSERT_LT(A, B) EXPECT_LT(A, B) +#define ASSERT_FALSE(X) EXPECT_FALSE(X) +#define ASSERT_TRUE(X) EXPECT_TRUE(X) +#define ASSERT_THROW(A, B) EXPECT_THROW(A, B) +#define ASSERT_NO_THROW(A) EXPECT_NO_THROW(A) +#define ASSERT_DOUBLE_EQ(A, B) EXPECT_DOUBLE_EQ(A, B) +#define ASSERT_STREQ(A, B) EXPECT_STREQ(A, B) diff --git a/library/cpp/testing/unittest/plugin.cpp b/library/cpp/testing/unittest/plugin.cpp index 543112f7ac..f03484a089 100644 --- a/library/cpp/testing/unittest/plugin.cpp +++ b/library/cpp/testing/unittest/plugin.cpp @@ -46,5 +46,5 @@ namespace NUnitTest { TPlugins::Instance().OnStopMain(argc, argv); } - } -} + } +} diff --git a/library/cpp/testing/unittest/plugin.h b/library/cpp/testing/unittest/plugin.h index 102f2c1469..f030654f97 100644 --- a/library/cpp/testing/unittest/plugin.h +++ b/library/cpp/testing/unittest/plugin.h @@ -25,5 +25,5 @@ namespace NUnitTest { TPluginRegistrator(TSimpleSharedPtr<IPlugin> plugin); }; - } -} + } +} diff --git a/library/cpp/testing/unittest/registar.cpp b/library/cpp/testing/unittest/registar.cpp index 3679b768ed..606e18cefd 100644 --- a/library/cpp/testing/unittest/registar.cpp +++ b/library/cpp/testing/unittest/registar.cpp @@ -11,27 +11,27 @@ #include <util/system/tls.h> #include <util/system/error.h> #include <util/string/cast.h> - + bool NUnitTest::ShouldColorizeDiff = true; bool NUnitTest::ContinueOnFail = false; TString NUnitTest::RandomString(size_t len, ui32 seed) { - TReallyFastRng32 rand(seed); + TReallyFastRng32 rand(seed); TString ret; - - ret.reserve(len); - - for (size_t i = 0; i < len; ++i) { - ret.push_back(char(rand.Uniform(1, 128))); - } - - return ret; -} - -Y_POD_STATIC_THREAD(bool) -UnittestThread; -Y_POD_STATIC_THREAD(NUnitTest::TTestBase*) -currentTest; + + ret.reserve(len); + + for (size_t i = 0; i < len; ++i) { + ret.push_back(char(rand.Uniform(1, 128))); + } + + return ret; +} + +Y_POD_STATIC_THREAD(bool) +UnittestThread; +Y_POD_STATIC_THREAD(NUnitTest::TTestBase*) +currentTest; ::NUnitTest::TRaiseErrorHandler RaiseErrorHandler; void ::NUnitTest::NPrivate::RaiseError(const char* what, const TString& msg, bool fatalFailure) { @@ -78,8 +78,8 @@ struct TDiffColorizer { explicit TDiffColorizer(bool reverse = false) : Reverse(reverse) - { - } + { + } TString Special(TStringBuf str) const { return ToString(Colors.YellowColor()) + str; @@ -111,8 +111,8 @@ struct TTraceDiffFormatter { explicit TTraceDiffFormatter(bool reverse = false) : Reverse(reverse) - { - } + { + } TString Special(TStringBuf str) const { return ToString(str); @@ -124,14 +124,14 @@ struct TTraceDiffFormatter { TString Left(TArrayRef<const char> str) const { return NUnitTest::GetFormatTag("good") + - TString(str.begin(), str.end()) + - NUnitTest::GetResetTag(); + TString(str.begin(), str.end()) + + NUnitTest::GetResetTag(); } TString Right(TArrayRef<const char> str) const { return NUnitTest::GetFormatTag("bad") + - TString(str.begin(), str.end()) + - NUnitTest::GetResetTag(); + TString(str.begin(), str.end()) + + NUnitTest::GetResetTag(); } }; @@ -149,7 +149,7 @@ TString NUnitTest::ColoredDiff(TStringBuf s1, TStringBuf s2, const TString& deli NDiff::InlineDiff(chunks, s1, s2, delims); if (NUnitTest::ShouldColorizeDiff) { NDiff::PrintChunks(res, TDiffColorizer(reverse), chunks); - } else { + } else { res << NUnitTest::GetResetTag(); NDiff::PrintChunks(res, TTraceDiffFormatter(reverse), chunks); } @@ -290,10 +290,10 @@ void NUnitTest::ITestBaseFactory::Register() noexcept { } NUnitTest::TTestBase::TTestBase() noexcept - : Parent_(nullptr) - , TestErrors_() - , CurrentSubtest_() -{ + : Parent_(nullptr) + , TestErrors_() + , CurrentSubtest_() +{ } NUnitTest::TTestBase::~TTestBase() = default; diff --git a/library/cpp/testing/unittest/registar.h b/library/cpp/testing/unittest/registar.h index 44517a0092..3307ba7e55 100644 --- a/library/cpp/testing/unittest/registar.h +++ b/library/cpp/testing/unittest/registar.h @@ -1,20 +1,20 @@ #pragma once - + #include <library/cpp/dbg_output/dump.h> - + #include <util/generic/bt_exception.h> #include <util/generic/hash.h> #include <util/generic/intrlist.h> #include <util/generic/map.h> #include <util/generic/ptr.h> #include <util/generic/set.h> -#include <util/generic/typetraits.h> +#include <util/generic/typetraits.h> #include <util/generic/vector.h> #include <util/generic/yexception.h> - -#include <util/string/builder.h> -#include <util/string/cast.h> -#include <util/string/printf.h> + +#include <util/string/builder.h> +#include <util/string/cast.h> +#include <util/string/printf.h> #include <util/system/defaults.h> #include <util/system/type_name.h> @@ -23,21 +23,21 @@ #include <util/system/rusage.h> -#include <cmath> -#include <cstdio> +#include <cmath> +#include <cstdio> #include <functional> extern bool CheckExceptionMessage(const char*, TString&); + +namespace NUnitTest { + class TTestBase; -namespace NUnitTest { - class TTestBase; - - namespace NPrivate { + namespace NPrivate { void RaiseError(const char* what, const TString& msg, bool fatalFailure); - void SetUnittestThread(bool); + void SetUnittestThread(bool); void SetCurrentTest(TTestBase*); TTestBase* GetCurrentTest(); - } + } extern bool ShouldColorizeDiff; extern bool ContinueOnFail; @@ -64,13 +64,13 @@ namespace NUnitTest { struct TTestContext { TTestContext() - : Processor(nullptr) - { + : Processor(nullptr) + { } explicit TTestContext(ITestSuiteProcessor* processor) - : Processor(processor) - { + : Processor(processor) + { } using TMetrics = THashMap<TString, double>; @@ -79,59 +79,59 @@ namespace NUnitTest { ITestSuiteProcessor* Processor; }; - class ITestSuiteProcessor { - public: - struct TUnit { + class ITestSuiteProcessor { + public: + struct TUnit { const TString name; - }; - - struct TTest { + }; + + struct TTest { const TUnit* unit; const char* name; - }; - - struct TError { + }; + + struct TError { const TTest* test; const char* msg; TString BackTrace; TTestContext* Context; - }; - + }; + struct TFinish { const TTest* test; TTestContext* Context; bool Success; - }; - + }; + ITestSuiteProcessor(); - + virtual ~ITestSuiteProcessor(); - + void Start(); - + void End(); - + void UnitStart(const TUnit& unit); - + void UnitStop(const TUnit& unit); - + void Error(const TError& descr); - + void BeforeTest(const TTest& test); - + void Finish(const TFinish& descr); - + unsigned GoodTests() const noexcept; unsigned FailTests() const noexcept; - + unsigned GoodTestsInCurrentUnit() const noexcept; - + unsigned FailTestsInCurrentUnit() const noexcept; - + // Should execute test suite? virtual bool CheckAccess(TString /*name*/, size_t /*num*/); - + // Should execute a test whitin suite? virtual bool CheckAccessTest(TString /*suite*/, const char* /*name*/); @@ -143,142 +143,142 @@ namespace NUnitTest { // --fork-tests is set (warning: this may be false, but never the less test will be forked if called inside UNIT_FORKED_TEST) virtual bool GetForkTests() const; - private: + private: virtual void OnStart(); - + virtual void OnEnd(); - + virtual void OnUnitStart(const TUnit* /*unit*/); - + virtual void OnUnitStop(const TUnit* /*unit*/); - + virtual void OnError(const TError* /*error*/); - + virtual void OnFinish(const TFinish* /*finish*/); - + virtual void OnBeforeTest(const TTest* /*test*/); void AddTestError(const TTest& test); void AddTestFinish(const TTest& test); - private: + private: TMap<TString, size_t> TestErrors_; TMap<TString, size_t> CurTestErrors_; - }; - - class TTestBase; - class TTestFactory; - - class ITestBaseFactory: public TIntrusiveListItem<ITestBaseFactory> { - public: + }; + + class TTestBase; + class TTestFactory; + + class ITestBaseFactory: public TIntrusiveListItem<ITestBaseFactory> { + public: ITestBaseFactory(); - + virtual ~ITestBaseFactory(); - + // name of test suite virtual TString Name() const noexcept = 0; - virtual TTestBase* ConstructTest() = 0; - - private: + virtual TTestBase* ConstructTest() = 0; + + private: void Register() noexcept; - }; - - class TTestBase { - friend class TTestFactory; + }; + + class TTestBase { + friend class TTestFactory; TRusage rusage; - - public: + + public: TTestBase() noexcept; - + virtual ~TTestBase(); - + virtual TString TypeId() const; - + virtual TString Name() const noexcept = 0; - virtual void Execute() = 0; - + virtual void Execute() = 0; + virtual void SetUp(); - + virtual void TearDown(); - + void AddError(const char* msg, const TString& backtrace = TString(), TTestContext* context = nullptr); void AddError(const char* msg, TTestContext* context); void RunAfterTest(std::function<void()> f); // function like atexit to run after current unit test - protected: + protected: bool CheckAccessTest(const char* test); void BeforeTest(const char* func); void Finish(const char* func, TTestContext* context); - + void AtStart(); - + void AtEnd(); - + void Run(std::function<void()> f, const TString& suite, const char* name, bool forceFork); - - class TCleanUp { - public: + + class TCleanUp { + public: explicit TCleanUp(TTestBase* base); - + ~TCleanUp(); - - private: - TTestBase* Base_; - }; - + + private: + TTestBase* Base_; + }; + void BeforeTest(); - + void AfterTest(); - + bool GetIsForked() const; bool GetForkTests() const; ITestSuiteProcessor* Processor() const noexcept; - - private: - TTestFactory* Parent_; + + private: + TTestFactory* Parent_; size_t TestErrors_; const char* CurrentSubtest_; TAdaptiveLock AfterTestFunctionsLock_; TVector<std::function<void()>> AfterTestFunctions_; - }; - -#define UNIT_TEST_SUITE(N) \ - typedef N TThisUnitTestSuite; \ - \ -public: \ - static TString StaticName() noexcept { \ - return TString(#N); \ - } \ - \ -private: \ - virtual TString Name() const noexcept override { \ - return this->StaticName(); \ - } \ - \ - virtual void Execute() override { \ - this->AtStart(); - -#define UNIT_TEST_SUITE_DEMANGLE(N) \ - typedef N TThisUnitTestSuite; \ - \ -public: \ - static TString StaticName() noexcept { \ + }; + +#define UNIT_TEST_SUITE(N) \ + typedef N TThisUnitTestSuite; \ + \ +public: \ + static TString StaticName() noexcept { \ + return TString(#N); \ + } \ + \ +private: \ + virtual TString Name() const noexcept override { \ + return this->StaticName(); \ + } \ + \ + virtual void Execute() override { \ + this->AtStart(); + +#define UNIT_TEST_SUITE_DEMANGLE(N) \ + typedef N TThisUnitTestSuite; \ + \ +public: \ + static TString StaticName() noexcept { \ return TypeName<N>(); \ - } \ - \ -private: \ - virtual TString Name() const noexcept override { \ - return this->StaticName(); \ - } \ - \ - virtual void Execute() override { \ + } \ + \ +private: \ + virtual TString Name() const noexcept override { \ + return this->StaticName(); \ + } \ + \ + virtual void Execute() override { \ this->AtStart(); #ifndef UT_SKIP_EXCEPTIONS @@ -289,21 +289,21 @@ private: \ #define CATCH_REACTION_BT(FN, e, context) throw #endif -#define UNIT_TEST_CHECK_TEST_IS_DECLARED_ONLY_ONCE(F) \ +#define UNIT_TEST_CHECK_TEST_IS_DECLARED_ONLY_ONCE(F) \ /* If you see this message - delete multiple UNIT_TEST(TestName) with same TestName. */ \ - /* It's forbidden to declare same test twice because it breaks --fork-tests logic. */ \ - int You_have_declared_test_##F##_multiple_times_This_is_forbidden; \ + /* It's forbidden to declare same test twice because it breaks --fork-tests logic. */ \ + int You_have_declared_test_##F##_multiple_times_This_is_forbidden; \ Y_UNUSED(You_have_declared_test_##F##_multiple_times_This_is_forbidden); -#define UNIT_TEST_RUN(F, FF, context) \ - this->BeforeTest((#F)); \ - { \ - struct T##F##Caller { \ - static void X(TThisUnitTestSuite* thiz, NUnitTest::TTestContext&) { \ - TCleanUp cleaner(thiz); \ - thiz->F(); \ - } \ - }; \ +#define UNIT_TEST_RUN(F, FF, context) \ + this->BeforeTest((#F)); \ + { \ + struct T##F##Caller { \ + static void X(TThisUnitTestSuite* thiz, NUnitTest::TTestContext&) { \ + TCleanUp cleaner(thiz); \ + thiz->F(); \ + } \ + }; \ this->TTestBase::Run(std::bind(&T##F##Caller::X, this, context), StaticName(), (#F), FF); \ } @@ -323,17 +323,17 @@ private: \ } \ this->Finish((#F), &context); \ } \ - } - + } + #define UNIT_TEST(F) UNIT_TEST_IMPL(F, false) - + #define UNIT_FORKED_TEST(F) UNIT_TEST_IMPL(F, true) -#define UNIT_TEST_EXCEPTION(F, E) \ - /* main process with "--fork-tests" flag treats exceptions as errors - it's result of forked test run */ \ - if (this->GetForkTests() && !this->GetIsForked()) { \ - UNIT_TEST_IMPL(F, false); \ - /* forked process (or main without "--fork-tests") treats some exceptions as success - it's exception test! */ \ +#define UNIT_TEST_EXCEPTION(F, E) \ + /* main process with "--fork-tests" flag treats exceptions as errors - it's result of forked test run */ \ + if (this->GetForkTests() && !this->GetIsForked()) { \ + UNIT_TEST_IMPL(F, false); \ + /* forked process (or main without "--fork-tests") treats some exceptions as success - it's exception test! */ \ } else { \ NUnitTest::TTestContext context(this->TTestBase::Processor()); \ if (this->CheckAccessTest((#F))) { \ @@ -342,7 +342,7 @@ private: \ this->AddError("exception expected", &context); \ } catch (const ::NUnitTest::TAssertException&) { \ } catch (const E& e) { \ - TString err; \ + TString err; \ if (!CheckExceptionMessage(e.what(), err)) \ this->AddError(err.c_str(), &context); \ } catch (const std::exception& e) { \ @@ -353,27 +353,27 @@ private: \ this->Finish((#F), &context); \ } \ } - -#define UNIT_TEST_SUITE_END() \ - this->AtEnd(); \ - } \ - \ -public: \ - /*for ; after macros*/ void sub##F() - -#define UNIT_FAIL_IMPL(R, M) \ - do { \ + +#define UNIT_TEST_SUITE_END() \ + this->AtEnd(); \ + } \ + \ +public: \ + /*for ; after macros*/ void sub##F() + +#define UNIT_FAIL_IMPL(R, M) \ + do { \ ::NUnitTest::NPrivate::RaiseError(R, ::TStringBuilder() << R << " at " << __LOCATION__ << ", " << __PRETTY_FUNCTION__ << ": " << M, true); \ } while (false) - + #define UNIT_FAIL_NONFATAL_IMPL(R, M) \ do { \ ::NUnitTest::NPrivate::RaiseError(R, ::TStringBuilder() << R << " at " << __LOCATION__ << ", " << __PRETTY_FUNCTION__ << ": " << M, false); \ } while (false) -#define UNIT_FAIL(M) UNIT_FAIL_IMPL("forced failure", M) +#define UNIT_FAIL(M) UNIT_FAIL_IMPL("forced failure", M) #define UNIT_FAIL_NONFATAL(M) UNIT_FAIL_NONFATAL_IMPL("forced failure", M) - + //types #define UNIT_ASSERT_TYPES_EQUAL(A, B) \ do { \ @@ -395,7 +395,7 @@ public: \ UNIT_FAIL_IMPL("assertion failure", failMsg); \ } \ } while (false) - + #define UNIT_ASSERT_DOUBLES_EQUAL_DEPRECATED(E, A, D) UNIT_ASSERT_DOUBLES_EQUAL_DEPRECATED_C(E, A, D, "") #define UNIT_ASSERT_DOUBLES_EQUAL_C(E, A, D, C) \ @@ -423,24 +423,24 @@ public: \ } while (false) #define UNIT_ASSERT_DOUBLES_EQUAL(E, A, D) UNIT_ASSERT_DOUBLES_EQUAL_C(E, A, D, "") - + //strings -#define UNIT_ASSERT_STRINGS_EQUAL_C(A, B, C) \ - do { \ - const TString _a(A); \ - const TString _b(B); \ - if (_a != _b) { \ +#define UNIT_ASSERT_STRINGS_EQUAL_C(A, B, C) \ + do { \ + const TString _a(A); \ + const TString _b(B); \ + if (_a != _b) { \ auto&& failMsg = Sprintf("%s != %s %s", ToString(_a).data(), ToString(_b).data(), (::TStringBuilder() << C).data()); \ UNIT_FAIL_IMPL("strings equal assertion failed", failMsg); \ - } \ + } \ } while (false) - + #define UNIT_ASSERT_STRINGS_EQUAL(A, B) UNIT_ASSERT_STRINGS_EQUAL_C(A, B, "") #define UNIT_ASSERT_STRING_CONTAINS_C(A, B, C) \ do { \ - const TString _a(A); \ - const TString _b(B); \ + const TString _a(A); \ + const TString _b(B); \ if (!_a.Contains(_b)) { \ auto&& msg = Sprintf("\"%s\" does not contain \"%s\", %s", ToString(_a).data(), ToString(_b).data(), (::TStringBuilder() << C).data()); \ UNIT_FAIL_IMPL("strings contains assertion failed", msg); \ @@ -449,26 +449,26 @@ public: \ #define UNIT_ASSERT_STRING_CONTAINS(A, B) UNIT_ASSERT_STRING_CONTAINS_C(A, B, "") -#define UNIT_ASSERT_NO_DIFF(A, B) \ - do { \ - const TString _a(A); \ - const TString _b(B); \ - if (_a != _b) { \ +#define UNIT_ASSERT_NO_DIFF(A, B) \ + do { \ + const TString _a(A); \ + const TString _b(B); \ + if (_a != _b) { \ UNIT_FAIL_IMPL("strings (" #A ") and (" #B ") are different", Sprintf("\n%s", ::NUnitTest::ColoredDiff(_a, _b, " \t\n.,:;'\"").data())); \ - } \ + } \ } while (false) //strings -#define UNIT_ASSERT_STRINGS_UNEQUAL_C(A, B, C) \ - do { \ - const TString _a(A); \ - const TString _b(B); \ - if (_a == _b) { \ +#define UNIT_ASSERT_STRINGS_UNEQUAL_C(A, B, C) \ + do { \ + const TString _a(A); \ + const TString _b(B); \ + if (_a == _b) { \ auto&& msg = Sprintf("%s == %s %s", ToString(_a).data(), ToString(_b).data(), (::TStringBuilder() << C).data()); \ - UNIT_FAIL_IMPL("strings unequal assertion failed", msg); \ - } \ + UNIT_FAIL_IMPL("strings unequal assertion failed", msg); \ + } \ } while (false) - + #define UNIT_ASSERT_STRINGS_UNEQUAL(A, B) UNIT_ASSERT_STRINGS_UNEQUAL_C(A, B, "") //bool @@ -478,9 +478,9 @@ public: \ UNIT_FAIL_IMPL("assertion failed", Sprintf("(%s) %s", #A, (::TStringBuilder() << C).data())); \ } \ } while (false) - + #define UNIT_ASSERT(A) UNIT_ASSERT_C(A, "") - + //general #define UNIT_ASSERT_EQUAL_C(A, B, C) \ do { \ @@ -488,16 +488,16 @@ public: \ UNIT_FAIL_IMPL("equal assertion failed", Sprintf("%s == %s %s", #A, #B, (::TStringBuilder() << C).data())); \ } \ } while (false) - + #define UNIT_ASSERT_EQUAL(A, B) UNIT_ASSERT_EQUAL_C(A, B, "") - + #define UNIT_ASSERT_UNEQUAL_C(A, B, C) \ do { \ if ((A) == (B)) { \ UNIT_FAIL_IMPL("unequal assertion failed", Sprintf("%s != %s %s", #A, #B, (::TStringBuilder() << C).data()));\ } \ } while (false) - + #define UNIT_ASSERT_UNEQUAL(A, B) UNIT_ASSERT_UNEQUAL_C(A, B, "") #define UNIT_ASSERT_LT_C(A, B, C) \ @@ -644,86 +644,86 @@ public: \ #define UNIT_ASSERT_NO_EXCEPTION_RESULT(A) UNIT_ASSERT_NO_EXCEPTION_RESULT_C(A, "") -#define UNIT_ASSERT_NO_EXCEPTION_C(A, C) \ - do { \ - try { \ - (void)(A); \ - } catch (const ::NUnitTest::TAssertException&) { \ - throw; \ - } catch (...) { \ +#define UNIT_ASSERT_NO_EXCEPTION_C(A, C) \ + do { \ + try { \ + (void)(A); \ + } catch (const ::NUnitTest::TAssertException&) { \ + throw; \ + } catch (...) { \ UNIT_FAIL_IMPL("exception-free assertion failed", Sprintf("%s throws %s\nException message: %s", #A, (::TStringBuilder() << C).data(), CurrentExceptionMessage().data())); \ - } \ + } \ } while (false) #define UNIT_ASSERT_NO_EXCEPTION(A) UNIT_ASSERT_NO_EXCEPTION_C(A, "") - namespace NPrivate { - template <class T, class U, bool Integers> - struct TCompareValuesImpl { - static inline bool Compare(const T& a, const U& b) { - return a == b; - } - }; - - template <class T, class U> - struct TCompareValuesImpl<T, U, true> { - static inline bool Compare(const T& a, const U& b) { - return ::ToString(a) == ::ToString(b); - } - }; - - template <class T, class U> + namespace NPrivate { + template <class T, class U, bool Integers> + struct TCompareValuesImpl { + static inline bool Compare(const T& a, const U& b) { + return a == b; + } + }; + + template <class T, class U> + struct TCompareValuesImpl<T, U, true> { + static inline bool Compare(const T& a, const U& b) { + return ::ToString(a) == ::ToString(b); + } + }; + + template <class T, class U> using TCompareValues = TCompareValuesImpl<T, U, std::is_integral<T>::value && std::is_integral<U>::value>; - - template <typename T, typename U> - static inline bool CompareEqual(const T& a, const U& b) { - return TCompareValues<T, U>::Compare(a, b); - } - - static inline bool CompareEqual(const char* a, const char* b) { - return 0 == strcmp(a, b); - } - - // helper method to avoid double evaluation of A and B expressions in UNIT_ASSERT_VALUES_EQUAL_C - template <typename T, typename U> + + template <typename T, typename U> + static inline bool CompareEqual(const T& a, const U& b) { + return TCompareValues<T, U>::Compare(a, b); + } + + static inline bool CompareEqual(const char* a, const char* b) { + return 0 == strcmp(a, b); + } + + // helper method to avoid double evaluation of A and B expressions in UNIT_ASSERT_VALUES_EQUAL_C + template <typename T, typename U> static inline bool CompareAndMakeStrings(const T& a, const U& b, TString& as, TString& asInd, TString& bs, TString& bsInd, bool& usePlainDiff, bool want) { - const bool have = CompareEqual(a, b); + const bool have = CompareEqual(a, b); usePlainDiff = std::is_integral<T>::value && std::is_integral<U>::value; - - if (want == have) { - return true; - } - - as = ::TStringBuilder() << ::DbgDump(a); - bs = ::TStringBuilder() << ::DbgDump(b); - asInd = ::TStringBuilder() << ::DbgDump(a).SetIndent(true); - bsInd = ::TStringBuilder() << ::DbgDump(b).SetIndent(true); - - return false; - } + + if (want == have) { + return true; + } + + as = ::TStringBuilder() << ::DbgDump(a); + bs = ::TStringBuilder() << ::DbgDump(b); + asInd = ::TStringBuilder() << ::DbgDump(a).SetIndent(true); + bsInd = ::TStringBuilder() << ::DbgDump(b).SetIndent(true); + + return false; + } } //values -#define UNIT_ASSERT_VALUES_EQUAL_IMPL(A, B, C, EQflag, EQstr, NEQstr) \ - do { \ - TString _as; \ - TString _bs; \ - TString _asInd; \ - TString _bsInd; \ - bool _usePlainDiff; \ - if (!::NUnitTest::NPrivate::CompareAndMakeStrings(A, B, _as, _asInd, _bs, _bsInd, _usePlainDiff, EQflag)) { \ +#define UNIT_ASSERT_VALUES_EQUAL_IMPL(A, B, C, EQflag, EQstr, NEQstr) \ + do { \ + TString _as; \ + TString _bs; \ + TString _asInd; \ + TString _bsInd; \ + bool _usePlainDiff; \ + if (!::NUnitTest::NPrivate::CompareAndMakeStrings(A, B, _as, _asInd, _bs, _bsInd, _usePlainDiff, EQflag)) { \ auto&& failMsg = Sprintf("(%s %s %s) failed: (%s %s %s) %s", #A, EQstr, #B, _as.data(), NEQstr, _bs.data(), (::TStringBuilder() << C).data()); \ - if (EQflag && !_usePlainDiff) { \ + if (EQflag && !_usePlainDiff) { \ failMsg += ", with diff:\n"; \ failMsg += ::NUnitTest::ColoredDiff(_asInd, _bsInd); \ - } \ + } \ UNIT_FAIL_IMPL("assertion failed", failMsg); \ - } \ + } \ } while (false) #define UNIT_ASSERT_VALUES_EQUAL_C(A, B, C) \ UNIT_ASSERT_VALUES_EQUAL_IMPL(A, B, C, true, "==", "!=") - + #define UNIT_ASSERT_VALUES_UNEQUAL_C(A, B, C) \ UNIT_ASSERT_VALUES_EQUAL_IMPL(A, B, C, false, "!=", "==") @@ -742,7 +742,7 @@ public: \ UNIT_FAIL_IMPL("fail test assertion failure", \ "code is expected to generate test failure, " \ "but it throws exception with message: " \ - << CurrentExceptionMessage()); \ + << CurrentExceptionMessage()); \ } \ if (!checker.Failed()) { \ UNIT_FAIL_IMPL("fail test assertion failure", \ @@ -754,46 +754,46 @@ public: \ #define UNIT_ADD_METRIC(name, value) ut_context.Metrics[name] = value - class TTestFactory { - friend class TTestBase; - friend class ITestBaseFactory; - - public: + class TTestFactory { + friend class TTestBase; + friend class ITestBaseFactory; + + public: static TTestFactory& Instance(); - + unsigned Execute(); - + void SetProcessor(ITestSuiteProcessor* processor); - - private: + + private: void Register(ITestBaseFactory* b) noexcept; - + ITestSuiteProcessor* Processor() const noexcept; - - private: + + private: explicit TTestFactory(ITestSuiteProcessor* processor); - + ~TTestFactory(); - - private: - TIntrusiveList<ITestBaseFactory> Items_; - ITestSuiteProcessor* Processor_; - }; - - template <class T> - class TTestBaseFactory: public ITestBaseFactory { - public: + + private: + TIntrusiveList<ITestBaseFactory> Items_; + ITestSuiteProcessor* Processor_; + }; + + template <class T> + class TTestBaseFactory: public ITestBaseFactory { + public: ~TTestBaseFactory() override = default; - + inline TTestBase* ConstructTest() override { - return new T; - } - + return new T; + } + inline TString Name() const noexcept override { return T::StaticName(); - } - }; - + } + }; + struct TBaseTestCase { // NOTE: since EACH test case is instantiated for listing tests, its // ctor/dtor are not the best place to do heavy preparations in test fixtures. @@ -802,15 +802,15 @@ public: \ inline TBaseTestCase() : TBaseTestCase(nullptr, nullptr, false) - { - } + { + } inline TBaseTestCase(const char* name, std::function<void(TTestContext&)> body, bool forceFork) : Name_(name) , Body_(std::move(body)) , ForceFork_(forceFork) - { - } + { + } virtual ~TBaseTestCase() = default; @@ -836,7 +836,7 @@ public: \ const char* Name_; std::function<void(TTestContext&)> Body_; bool ForceFork_; - }; + }; using TBaseFixture = TBaseTestCase; @@ -911,76 +911,76 @@ public: \ bool Failed_ = false; }; -#define UNIT_TEST_SUITE_REGISTRATION(T) \ +#define UNIT_TEST_SUITE_REGISTRATION(T) \ static const ::NUnitTest::TTestBaseFactory<T> Y_GENERATE_UNIQUE_ID(UTREG_); - + #define Y_UNIT_TEST_SUITE_IMPL_F(N, T, F) \ - namespace NTestSuite##N { \ + namespace NTestSuite##N { \ class TCurrentTestCase: public F { \ }; \ - class TCurrentTest: public T { \ - private: \ + class TCurrentTest: public T { \ + private: \ typedef std::function<THolder<NUnitTest::TBaseTestCase>()> TTestCaseFactory; \ typedef TVector<TTestCaseFactory> TTests; \ - \ - static TTests& Tests() { \ - static TTests tests; \ - return tests; \ - } \ - \ - public: \ - static TString StaticName() { \ - return #N; \ - } \ - virtual TString Name() const noexcept { \ - return StaticName(); \ - } \ - \ + \ + static TTests& Tests() { \ + static TTests tests; \ + return tests; \ + } \ + \ + public: \ + static TString StaticName() { \ + return #N; \ + } \ + virtual TString Name() const noexcept { \ + return StaticName(); \ + } \ + \ static void AddTest(const char* name, \ const std::function<void(NUnitTest::TTestContext&)>& body, bool forceFork) \ { \ Tests().push_back([=]{ return MakeHolder<NUnitTest::TBaseTestCase>(name, body, forceFork); }); \ - } \ - \ + } \ + \ static void AddTest(TTestCaseFactory testCaseFactory) { \ Tests().push_back(std::move(testCaseFactory)); \ } \ \ - virtual void Execute() { \ - this->AtStart(); \ + virtual void Execute() { \ + this->AtStart(); \ for (TTests::iterator it = Tests().begin(), ie = Tests().end(); it != ie; ++it) { \ const auto i = (*it)(); \ if (!this->CheckAccessTest(i->Name_)) { \ - continue; \ - } \ + continue; \ + } \ NUnitTest::TTestContext context(this->TTestBase::Processor()); \ - try { \ + try { \ this->BeforeTest(i->Name_); \ - { \ - TCleanUp cleaner(this); \ + { \ + TCleanUp cleaner(this); \ auto testCase = [&i, &context] { \ i->SetUp(context); \ i->Execute_(context); \ i->TearDown(context); \ }; \ this->T::Run(testCase, StaticName(), i->Name_, i->ForceFork_); \ - } \ - } catch (const ::NUnitTest::TAssertException&) { \ - } catch (const yexception& e) { \ + } \ + } catch (const ::NUnitTest::TAssertException&) { \ + } catch (const yexception& e) { \ CATCH_REACTION_BT(i->Name_, e, &context); \ - } catch (const std::exception& e) { \ + } catch (const std::exception& e) { \ CATCH_REACTION(i->Name_, e, &context); \ - } catch (...) { \ - this->AddError("non-std exception!", &context); \ - } \ + } catch (...) { \ + this->AddError("non-std exception!", &context); \ + } \ this->Finish(i->Name_, &context); \ - } \ - this->AtEnd(); \ - } \ - }; \ - UNIT_TEST_SUITE_REGISTRATION(TCurrentTest) \ - } \ - namespace NTestSuite##N + } \ + this->AtEnd(); \ + } \ + }; \ + UNIT_TEST_SUITE_REGISTRATION(TCurrentTest) \ + } \ + namespace NTestSuite##N #define Y_UNIT_TEST_SUITE_IMPL(N, T) Y_UNIT_TEST_SUITE_IMPL_F(N, T, ::NUnitTest::TBaseTestCase) #define Y_UNIT_TEST_SUITE(N) Y_UNIT_TEST_SUITE_IMPL(N, TTestBase) @@ -1016,7 +1016,7 @@ public: \ #define SIMPLE_UNIT_FORKED_TEST(N) Y_UNIT_TEST_IMPL(N, true, TCurrentTestCase) #define Y_UNIT_TEST_SUITE_IMPLEMENTATION(N) \ - namespace NTestSuite##N + namespace NTestSuite##N #define Y_UNIT_TEST_DECLARE(N) \ struct TTestCase##N @@ -1025,6 +1025,6 @@ public: \ friend NTestSuite##N::TTestCase##T \ TString RandomString(size_t len, ui32 seed = 0); -} - +} + using ::NUnitTest::TTestBase; diff --git a/library/cpp/testing/unittest/tests_data.cpp b/library/cpp/testing/unittest/tests_data.cpp index b51cbc4b87..08a4eb0790 100644 --- a/library/cpp/testing/unittest/tests_data.cpp +++ b/library/cpp/testing/unittest/tests_data.cpp @@ -11,7 +11,7 @@ public: TPortManagerImpl(bool reservePortsForCurrentTest) : EnableReservePortsForCurrentTest(reservePortsForCurrentTest) , DisableRandomPorts(!GetEnv("NO_RANDOM_PORTS").empty()) - { + { } ui16 GetPort(ui16 port) { diff --git a/library/cpp/testing/unittest/tests_data.h b/library/cpp/testing/unittest/tests_data.h index 6536bc1ae6..2c56bdb3b2 100644 --- a/library/cpp/testing/unittest/tests_data.h +++ b/library/cpp/testing/unittest/tests_data.h @@ -27,7 +27,7 @@ void SetReuseAddressAndPort(const TSocketType& sock) { #endif } -class TPortManager: public TNonCopyable { +class TPortManager: public TNonCopyable { public: TPortManager(bool reservePortsForCurrentTest = true); ~TPortManager(); diff --git a/library/cpp/testing/unittest/ut/main.cpp b/library/cpp/testing/unittest/ut/main.cpp index e303e21e30..cba1681c83 100644 --- a/library/cpp/testing/unittest/ut/main.cpp +++ b/library/cpp/testing/unittest/ut/main.cpp @@ -7,27 +7,27 @@ #include <util/system/env.h> #include <util/system/fs.h> -TEST(GTest, Test1) { - UNIT_ASSERT_EQUAL(1, 1); -} - -TEST(GTest, Test2) { - UNIT_ASSERT_EQUAL(2, 2); -} - -namespace { - struct TFixture : ::testing::Test { - TFixture() - : I(0) - { - } - +TEST(GTest, Test1) { + UNIT_ASSERT_EQUAL(1, 1); +} + +TEST(GTest, Test2) { + UNIT_ASSERT_EQUAL(2, 2); +} + +namespace { + struct TFixture : ::testing::Test { + TFixture() + : I(0) + { + } + void SetUp() override { - I = 5; - } - - int I; - }; + I = 5; + } + + int I; + }; struct TSimpleFixture : public NUnitTest::TBaseFixture { size_t Value = 24; @@ -50,11 +50,11 @@ namespace { Magic = 100; } }; -} - -TEST_F(TFixture, Test1) { - ASSERT_EQ(I, 5); -} +} + +TEST_F(TFixture, Test1) { + ASSERT_EQ(I, 5); +} TEST(ETest, Test1) { UNIT_CHECK_GENERATED_EXCEPTION(ythrow yexception(), yexception); diff --git a/library/cpp/testing/unittest/utmain.cpp b/library/cpp/testing/unittest/utmain.cpp index 305bc6b40f..9b46bee6ee 100644 --- a/library/cpp/testing/unittest/utmain.cpp +++ b/library/cpp/testing/unittest/utmain.cpp @@ -1,9 +1,9 @@ #include "plugin.h" -#include "registar.h" -#include "utmain.h" - +#include "registar.h" +#include "utmain.h" + #include <library/cpp/colorizer/colors.h> - + #include <library/cpp/json/writer/json.h> #include <library/cpp/json/writer/json_value.h> #include <library/cpp/testing/common/env.h> @@ -11,45 +11,45 @@ #include <util/datetime/base.h> -#include <util/generic/hash.h> +#include <util/generic/hash.h> #include <util/generic/hash_set.h> #include <util/generic/scope.h> #include <util/generic/string.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> -#include <util/network/init.h> +#include <util/network/init.h> #include <util/stream/file.h> #include <util/stream/output.h> #include <util/string/join.h> #include <util/string/util.h> - + #include <util/system/defaults.h> #include <util/system/execpath.h> #include <util/system/valgrind.h> #include <util/system/shellcommand.h> -#if defined(_win_) -#include <fcntl.h> -#include <io.h> -#include <windows.h> -#include <crtdbg.h> -#endif - -#if defined(_unix_) -#include <unistd.h> -#endif - +#if defined(_win_) +#include <fcntl.h> +#include <io.h> +#include <windows.h> +#include <crtdbg.h> +#endif + +#if defined(_unix_) +#include <unistd.h> +#endif + #ifdef WITH_VALGRIND #define NOTE_IN_VALGRIND(test) VALGRIND_PRINTF("%s::%s", test->unit->name.data(), test->name) #else -#define NOTE_IN_VALGRIND(test) +#define NOTE_IN_VALGRIND(test) #endif const size_t MAX_COMMENT_MESSAGE_LENGTH = 1024 * 1024; // 1 MB -using namespace NUnitTest; - +using namespace NUnitTest; + class TNullTraceWriterProcessor: public ITestSuiteProcessor { }; @@ -107,12 +107,12 @@ private: virtual TString BuildComment(const char* message, const char* backTrace) { return NUnitTest::GetFormatTag("bad") + - TString(message).substr(0, MAX_COMMENT_MESSAGE_LENGTH) + - NUnitTest::GetResetTag() + - TString("\n") + - NUnitTest::GetFormatTag("alt1") + - TString(backTrace).substr(0, MAX_COMMENT_MESSAGE_LENGTH) + - NUnitTest::GetResetTag(); + TString(message).substr(0, MAX_COMMENT_MESSAGE_LENGTH) + + NUnitTest::GetResetTag() + + TString("\n") + + NUnitTest::GetFormatTag("alt1") + + TString(backTrace).substr(0, MAX_COMMENT_MESSAGE_LENGTH) + + NUnitTest::GetResetTag(); } void OnBeforeTest(const TTest* test) override { @@ -164,75 +164,75 @@ private: } }; -class TColoredProcessor: public ITestSuiteProcessor, public NColorizer::TColors { -public: +class TColoredProcessor: public ITestSuiteProcessor, public NColorizer::TColors { +public: inline TColoredProcessor(const TString& appName) - : PrintBeforeSuite_(true) - , PrintBeforeTest_(true) + : PrintBeforeSuite_(true) + , PrintBeforeTest_(true) , PrintAfterTest_(true) , PrintAfterSuite_(true) - , PrintTimes_(false) + , PrintTimes_(false) , PrintSummary_(true) - , PrevTime_(TInstant::Now()) + , PrevTime_(TInstant::Now()) , ShowFails(true) - , Start(0) - , End(Max<size_t>()) - , AppName(appName) - , ForkTests(false) - , IsForked(false) - , Loop(false) - , ForkExitedCorrectly(false) + , Start(0) + , End(Max<size_t>()) + , AppName(appName) + , ForkTests(false) + , IsForked(false) + , Loop(false) + , ForkExitedCorrectly(false) , TraceProcessor(new TNullTraceWriterProcessor()) - { - } - + { + } + ~TColoredProcessor() override { - } - - inline void Disable(const char* name) { + } + + inline void Disable(const char* name) { size_t colon = TString(name).find("::"); if (colon == TString::npos) { - DisabledSuites_.insert(name); - } else { + DisabledSuites_.insert(name); + } else { TString suite = TString(name).substr(0, colon); - DisabledTests_.insert(name); - } - } - - inline void Enable(const char* name) { + DisabledTests_.insert(name); + } + } + + inline void Enable(const char* name) { size_t colon = TString(name).rfind("::"); if (colon == TString::npos) { - EnabledSuites_.insert(name); + EnabledSuites_.insert(name); EnabledTests_.insert(TString() + name + "::*"); - } else { + } else { TString suite = TString(name).substr(0, colon); - EnabledSuites_.insert(suite); + EnabledSuites_.insert(suite); EnabledSuites_.insert(name); - EnabledTests_.insert(name); + EnabledTests_.insert(name); EnabledTests_.insert(TString() + name + "::*"); - } - } - - inline void SetPrintBeforeSuite(bool print) { - PrintBeforeSuite_ = print; - } + } + } + + inline void SetPrintBeforeSuite(bool print) { + PrintBeforeSuite_ = print; + } inline void SetPrintAfterSuite(bool print) { PrintAfterSuite_ = print; } - inline void SetPrintBeforeTest(bool print) { - PrintBeforeTest_ = print; - } + inline void SetPrintBeforeTest(bool print) { + PrintBeforeTest_ = print; + } inline void SetPrintAfterTest(bool print) { PrintAfterTest_ = print; } - inline void SetPrintTimes(bool print) { - PrintTimes_ = print; - } - + inline void SetPrintTimes(bool print) { + PrintTimes_ = print; + } + inline void SetPrintSummary(bool print) { PrintSummary_ = print; } @@ -241,15 +241,15 @@ public: return PrintSummary_; } - inline void SetShowFails(bool show) { - ShowFails = show; - } + inline void SetShowFails(bool show) { + ShowFails = show; + } void SetContinueOnFail(bool val) { NUnitTest::ContinueOnFail = val; } - inline void BeQuiet() { + inline void BeQuiet() { SetPrintTimes(false); SetPrintBeforeSuite(false); SetPrintAfterSuite(false); @@ -258,117 +258,117 @@ public: SetPrintSummary(false); } - inline void SetStart(size_t val) { - Start = val; - } - - inline void SetEnd(size_t val) { - End = val; - } - - inline void SetForkTests(bool val) { - ForkTests = val; - } + inline void SetStart(size_t val) { + Start = val; + } + + inline void SetEnd(size_t val) { + End = val; + } + + inline void SetForkTests(bool val) { + ForkTests = val; + } inline bool GetForkTests() const override { - return ForkTests; - } + return ForkTests; + } - inline void SetIsForked(bool val) { - IsForked = val; - SetIsTTY(IsForked || CalcIsTTY(stderr)); - } + inline void SetIsForked(bool val) { + IsForked = val; + SetIsTTY(IsForked || CalcIsTTY(stderr)); + } inline bool GetIsForked() const override { - return IsForked; - } + return IsForked; + } - inline void SetLoop(bool loop) { - Loop = loop; - } - - inline bool IsLoop() const { - return Loop; - } + inline void SetLoop(bool loop) { + Loop = loop; + } + inline bool IsLoop() const { + return Loop; + } + inline void SetTraceProcessor(TAutoPtr<ITestSuiteProcessor> traceProcessor) { TraceProcessor = traceProcessor; } -private: +private: void OnUnitStart(const TUnit* unit) override { TraceProcessor->UnitStart(*unit); - if (IsForked) { - return; + if (IsForked) { + return; } - if (PrintBeforeSuite_ || PrintBeforeTest_) { + if (PrintBeforeSuite_ || PrintBeforeTest_) { fprintf(stderr, "%s<-----%s %s\n", LightBlueColor().data(), OldColor().data(), unit->name.data()); - } - } + } + } void OnUnitStop(const TUnit* unit) override { TraceProcessor->UnitStop(*unit); - if (IsForked) { - return; + if (IsForked) { + return; } if (!PrintAfterSuite_) { return; } - fprintf(stderr, "%s----->%s %s -> ok: %s%u%s", + fprintf(stderr, "%s----->%s %s -> ok: %s%u%s", LightBlueColor().data(), OldColor().data(), unit->name.data(), LightGreenColor().data(), GoodTestsInCurrentUnit(), OldColor().data()); - if (FailTestsInCurrentUnit()) { - fprintf(stderr, ", err: %s%u%s", + if (FailTestsInCurrentUnit()) { + fprintf(stderr, ", err: %s%u%s", LightRedColor().data(), FailTestsInCurrentUnit(), OldColor().data()); - } - fprintf(stderr, "\n"); - } - + } + fprintf(stderr, "\n"); + } + void OnBeforeTest(const TTest* test) override { TraceProcessor->BeforeTest(*test); - if (IsForked) { - return; + if (IsForked) { + return; } - if (PrintBeforeTest_) { + if (PrintBeforeTest_) { fprintf(stderr, "[%sexec%s] %s::%s...\n", LightBlueColor().data(), OldColor().data(), test->unit->name.data(), test->name); - } - } + } + } void OnError(const TError* descr) override { TraceProcessor->Error(*descr); - if (!IsForked && ForkExitedCorrectly) { - return; - } + if (!IsForked && ForkExitedCorrectly) { + return; + } if (!PrintAfterTest_) { return; } const TString err = Sprintf("[%sFAIL%s] %s::%s -> %s%s%s\n%s%s%s", LightRedColor().data(), OldColor().data(), descr->test->unit->name.data(), - descr->test->name, + descr->test->name, LightRedColor().data(), descr->msg, OldColor().data(), LightCyanColor().data(), descr->BackTrace.data(), OldColor().data()); const TDuration test_duration = SaveTestDuration(); - if (ShowFails) { + if (ShowFails) { if (PrintTimes_) { Fails.push_back(Sprintf("%s %s", test_duration.ToString().data(), err.data())); } else { Fails.push_back(err); } - } + } fprintf(stderr, "%s", err.data()); - NOTE_IN_VALGRIND(descr->test); + NOTE_IN_VALGRIND(descr->test); PrintTimes(test_duration); - if (IsForked) { - fprintf(stderr, "%s", ForkCorrectExitMsg); - } - } - + if (IsForked) { + fprintf(stderr, "%s", ForkCorrectExitMsg); + } + } + void OnFinish(const TFinish* descr) override { TraceProcessor->Finish(*descr); - if (!IsForked && ForkExitedCorrectly) { - return; - } + if (!IsForked && ForkExitedCorrectly) { + return; + } if (!PrintAfterTest_) { return; } @@ -382,9 +382,9 @@ private: if (IsForked) { fprintf(stderr, "%s", ForkCorrectExitMsg); } - } - } - + } + } + inline TDuration SaveTestDuration() { const TInstant now = TInstant::Now(); TDuration d = now - PrevTime_; @@ -393,101 +393,101 @@ private: } inline void PrintTimes(TDuration d) { - if (!PrintTimes_) { - return; - } - + if (!PrintTimes_) { + return; + } + Cerr << d << "\n"; - } - + } + void OnEnd() override { TraceProcessor->End(); - if (IsForked) { - return; - } + if (IsForked) { + return; + } if (!PrintSummary_) { return; } - fprintf(stderr, "[%sDONE%s] ok: %s%u%s", + fprintf(stderr, "[%sDONE%s] ok: %s%u%s", YellowColor().data(), OldColor().data(), LightGreenColor().data(), GoodTests(), OldColor().data()); - if (FailTests()) - fprintf(stderr, ", err: %s%u%s", + if (FailTests()) + fprintf(stderr, ", err: %s%u%s", LightRedColor().data(), FailTests(), OldColor().data()); - fprintf(stderr, "\n"); + fprintf(stderr, "\n"); - if (ShowFails) { - for (size_t i = 0; i < Fails.size(); ++i) { + if (ShowFails) { + for (size_t i = 0; i < Fails.size(); ++i) { printf("%s", Fails[i].data()); } - } - } - + } + } + bool CheckAccess(TString name, size_t num) override { - if (num < Start) { - return false; - } - - if (num >= End) { - return false; - } - + if (num < Start) { + return false; + } + + if (num >= End) { + return false; + } + if (DisabledSuites_.find(name.data()) != DisabledSuites_.end()) { - return false; - } - - if (EnabledSuites_.empty()) { - return true; - } - + return false; + } + + if (EnabledSuites_.empty()) { + return true; + } + return EnabledSuites_.find(name.data()) != EnabledSuites_.end(); - } - + } + bool CheckAccessTest(TString suite, const char* test) override { TString name = suite + "::" + test; - if (DisabledTests_.find(name) != DisabledTests_.end()) { - return false; - } - - if (EnabledTests_.empty()) { - return true; - } + if (DisabledTests_.find(name) != DisabledTests_.end()) { + return false; + } + + if (EnabledTests_.empty()) { + return true; + } if (EnabledTests_.find(TString() + suite + "::*") != EnabledTests_.end()) { - return true; - } + return true; + } - return EnabledTests_.find(name) != EnabledTests_.end(); - } + return EnabledTests_.find(name) != EnabledTests_.end(); + } void Run(std::function<void()> f, const TString& suite, const char* name, const bool forceFork) override { - if (!(ForkTests || forceFork) || GetIsForked()) { - return f(); + if (!(ForkTests || forceFork) || GetIsForked()) { + return f(); } TList<TString> args(1, "--is-forked-internal"); args.push_back(Sprintf("+%s::%s", suite.data(), name)); - // stdin is ignored - unittest should not need them... - TShellCommand cmd(AppName, args, - TShellCommandOptions().SetUseShell(false).SetCloseAllFdsOnExec(true).SetAsync(false).SetLatency(1)); - cmd.Run(); + // stdin is ignored - unittest should not need them... + TShellCommand cmd(AppName, args, + TShellCommandOptions().SetUseShell(false).SetCloseAllFdsOnExec(true).SetAsync(false).SetLatency(1)); + cmd.Run(); const TString& err = cmd.GetError(); - const size_t msgIndex = err.find(ForkCorrectExitMsg); + const size_t msgIndex = err.find(ForkCorrectExitMsg); - // everything is printed by parent process except test's result output ("good" or "fail") - // which is printed by child. If there was no output - parent process prints default message. + // everything is printed by parent process except test's result output ("good" or "fail") + // which is printed by child. If there was no output - parent process prints default message. ForkExitedCorrectly = msgIndex != TString::npos; - // TODO: stderr output is always printed after stdout - Cout.Write(cmd.GetOutput()); - Cerr.Write(err.c_str(), Min(msgIndex, err.size())); + // TODO: stderr output is always printed after stdout + Cout.Write(cmd.GetOutput()); + Cerr.Write(err.c_str(), Min(msgIndex, err.size())); - // do not use default case, so gcc will warn if new element in enum will be added - switch (cmd.GetStatus()) { + // do not use default case, so gcc will warn if new element in enum will be added + switch (cmd.GetStatus()) { case TShellCommand::SHELL_FINISHED: { // test could fail with zero status if it calls exit(0) in the middle. if (ForkExitedCorrectly) @@ -508,40 +508,40 @@ private: ythrow yexception() << "Forked test failed with internal error: " << cmd.GetInternalError(); } } - } + } -private: - bool PrintBeforeSuite_; - bool PrintBeforeTest_; +private: + bool PrintBeforeSuite_; + bool PrintBeforeTest_; bool PrintAfterTest_; bool PrintAfterSuite_; - bool PrintTimes_; + bool PrintTimes_; bool PrintSummary_; THashSet<TString> DisabledSuites_; THashSet<TString> EnabledSuites_; THashSet<TString> DisabledTests_; THashSet<TString> EnabledTests_; - TInstant PrevTime_; - bool ShowFails; + TInstant PrevTime_; + bool ShowFails; TVector<TString> Fails; - size_t Start; - size_t End; + size_t Start; + size_t End; TString AppName; - bool ForkTests; - bool IsForked; - bool Loop; - static const char* const ForkCorrectExitMsg; - bool ForkExitedCorrectly; + bool ForkTests; + bool IsForked; + bool Loop; + static const char* const ForkCorrectExitMsg; + bool ForkExitedCorrectly; TAutoPtr<ITestSuiteProcessor> TraceProcessor; -}; - -const char* const TColoredProcessor::ForkCorrectExitMsg = "--END--"; - +}; + +const char* const TColoredProcessor::ForkCorrectExitMsg = "--END--"; + class TEnumeratingProcessor: public ITestSuiteProcessor { public: TEnumeratingProcessor(bool verbose, IOutputStream& stream) noexcept - : Verbose_(verbose) - , Stream_(stream) + : Verbose_(verbose) + , Stream_(stream) { } @@ -571,7 +571,7 @@ private: class TWinEnvironment { public: TWinEnvironment() - : OutputCP(GetConsoleOutputCP()) + : OutputCP(GetConsoleOutputCP()) { setmode(fileno(stdout), _O_BINARY); SetConsoleOutputCP(CP_UTF8); @@ -593,7 +593,7 @@ public: SetConsoleOutputCP(OutputCP); // restore original output CP at program exit } - + private: UINT OutputCP; // original codepage }; @@ -648,15 +648,15 @@ int NUnitTest::RunMain(int argc, char** argv) { } #endif NTesting::THook::CallBeforeInit(); - InitNetworkSubSystem(); - - try { + InitNetworkSubSystem(); + + try { GetExecPath(); - } catch (...) { - } - + } catch (...) { + } + #ifndef UT_SKIP_EXCEPTIONS - try { + try { #endif NTesting::THook::CallBeforeRun(); Y_DEFER { NTesting::THook::CallAfterRun(); }; @@ -667,7 +667,7 @@ int NUnitTest::RunMain(int argc, char** argv) { TColoredProcessor processor(GetExecPath()); IOutputStream* listStream = &Cout; THolder<IOutputStream> listFile; - + enum EListType { DONT_LIST, LIST, @@ -675,17 +675,17 @@ int NUnitTest::RunMain(int argc, char** argv) { }; EListType listTests = DONT_LIST; - for (size_t i = 1; i < (size_t)argc; ++i) { - const char* name = argv[i]; - - if (name && *name) { + for (size_t i = 1; i < (size_t)argc; ++i) { + const char* name = argv[i]; + + if (name && *name) { if (strcmp(name, "--help") == 0 || strcmp(name, "-h") == 0) { return DoUsage(argv[0]); } else if (strcmp(name, "--list") == 0 || strcmp(name, "-l") == 0) { listTests = LIST; } else if (strcmp(name, "--list-verbose") == 0 || strcmp(name, "-A") == 0) { listTests = LIST_VERBOSE; - } else if (strcmp(name, "--print-before-suite=false") == 0) { + } else if (strcmp(name, "--print-before-suite=false") == 0) { processor.SetPrintBeforeSuite(false); } else if (strcmp(name, "--print-before-test=false") == 0) { processor.SetPrintBeforeTest(false); @@ -699,14 +699,14 @@ int NUnitTest::RunMain(int argc, char** argv) { processor.SetShowFails(false); } else if (strcmp(name, "--continue-on-fail") == 0) { processor.SetContinueOnFail(true); - } else if (strcmp(name, "--print-times") == 0) { - processor.SetPrintTimes(true); - } else if (strcmp(name, "--from") == 0) { - ++i; - processor.SetStart(FromString<size_t>(argv[i])); - } else if (strcmp(name, "--to") == 0) { - ++i; - processor.SetEnd(FromString<size_t>(argv[i])); + } else if (strcmp(name, "--print-times") == 0) { + processor.SetPrintTimes(true); + } else if (strcmp(name, "--from") == 0) { + ++i; + processor.SetStart(FromString<size_t>(argv[i])); + } else if (strcmp(name, "--to") == 0) { + ++i; + processor.SetEnd(FromString<size_t>(argv[i])); } else if (strcmp(name, "--fork-tests") == 0) { processor.SetForkTests(true); } else if (strcmp(name, "--is-forked-internal") == 0) { @@ -735,19 +735,19 @@ int NUnitTest::RunMain(int argc, char** argv) { } else if (TString(name).StartsWith("--")) { return DoUsage(argv[0]), 1; } else if (*name == '-') { - processor.Disable(name + 1); - } else if (*name == '+') { - processor.Enable(name + 1); - } else { - processor.Enable(name); - } - } + processor.Disable(name + 1); + } else if (*name == '+') { + processor.Enable(name + 1); + } else { + processor.Enable(name); + } + } } if (listTests != DONT_LIST) { return DoList(listTests == LIST_VERBOSE, *listStream); } - - TTestFactory::Instance().SetProcessor(&processor); + + TTestFactory::Instance().SetProcessor(&processor); unsigned ret; for (;;) { @@ -759,13 +759,13 @@ int NUnitTest::RunMain(int argc, char** argv) { if (0 != ret || !processor.IsLoop()) { break; } - } + } return ret; #ifndef UT_SKIP_EXCEPTIONS - } catch (...) { - Cerr << "caught exception in test suite(" << CurrentExceptionMessage() << ")" << Endl; - } + } catch (...) { + Cerr << "caught exception in test suite(" << CurrentExceptionMessage() << ")" << Endl; + } #endif - - return 1; -} + + return 1; +} diff --git a/library/cpp/testing/unittest/utmain.h b/library/cpp/testing/unittest/utmain.h index 65e8082ee1..07c288d5a5 100644 --- a/library/cpp/testing/unittest/utmain.h +++ b/library/cpp/testing/unittest/utmain.h @@ -1,5 +1,5 @@ -#pragma once - -namespace NUnitTest { - int RunMain(int argc, char** argv); -} +#pragma once + +namespace NUnitTest { + int RunMain(int argc, char** argv); +} diff --git a/library/cpp/testing/unittest/ya.make b/library/cpp/testing/unittest/ya.make index aaa4f2ba85..2d27845bb2 100644 --- a/library/cpp/testing/unittest/ya.make +++ b/library/cpp/testing/unittest/ya.make @@ -1,4 +1,4 @@ -LIBRARY() +LIBRARY() PROVIDES(test_framework) @@ -6,26 +6,26 @@ OWNER( pg galaxycrab ) - -PEERDIR( + +PEERDIR( library/cpp/colorizer library/cpp/dbg_output library/cpp/diff library/cpp/json/writer library/cpp/testing/common library/cpp/testing/hook -) - -SRCS( - gtest.cpp +) + +SRCS( + gtest.cpp checks.cpp plugin.cpp - registar.cpp + registar.cpp tests_data.cpp - utmain.cpp -) + utmain.cpp +) -END() +END() RECURSE_FOR_TESTS( fat diff --git a/library/cpp/testing/unittest_main/main.cpp b/library/cpp/testing/unittest_main/main.cpp index fc5d2d9418..e2cecd3a8f 100644 --- a/library/cpp/testing/unittest_main/main.cpp +++ b/library/cpp/testing/unittest_main/main.cpp @@ -1,5 +1,5 @@ #include <library/cpp/testing/unittest/utmain.h> - -int main(int argc, char** argv) { - return NUnitTest::RunMain(argc, argv); -} + +int main(int argc, char** argv) { + return NUnitTest::RunMain(argc, argv); +} diff --git a/library/cpp/testing/unittest_main/ya.make b/library/cpp/testing/unittest_main/ya.make index 80a6cc699b..43b36bde3f 100644 --- a/library/cpp/testing/unittest_main/ya.make +++ b/library/cpp/testing/unittest_main/ya.make @@ -1,14 +1,14 @@ -LIBRARY() - -OWNER(pg) - -PEERDIR( +LIBRARY() + +OWNER(pg) + +PEERDIR( library/cpp/testing/unittest library/cpp/terminate_handler -) - -SRCS( - main.cpp -) - -END() +) + +SRCS( + main.cpp +) + +END() diff --git a/library/cpp/threading/atomic/bool.h b/library/cpp/threading/atomic/bool.h index d52544e762..ec8f75427b 100644 --- a/library/cpp/threading/atomic/bool.h +++ b/library/cpp/threading/atomic/bool.h @@ -1,7 +1,7 @@ -#pragma once - +#pragma once + #include <util/system/atomic.h> - + namespace NAtomic { class TBool { public: @@ -20,12 +20,12 @@ namespace NAtomic { return AtomicGet(Val_); } - const TBool& operator=(bool val) noexcept { + const TBool& operator=(bool val) noexcept { AtomicSet(Val_, val); return *this; } - const TBool& operator=(const TBool& src) noexcept { + const TBool& operator=(const TBool& src) noexcept { AtomicSet(Val_, AtomicGet(src.Val_)); return *this; } @@ -33,4 +33,4 @@ namespace NAtomic { private: TAtomic Val_ = 0; }; -} +} diff --git a/library/cpp/threading/chunk_queue/queue.h b/library/cpp/threading/chunk_queue/queue.h index 55859601a1..1959b0258c 100644 --- a/library/cpp/threading/chunk_queue/queue.h +++ b/library/cpp/threading/chunk_queue/queue.h @@ -19,21 +19,21 @@ namespace NThreading { // Platform helpers #if !defined(PLATFORM_CACHE_LINE) -#define PLATFORM_CACHE_LINE 64 +#define PLATFORM_CACHE_LINE 64 #endif #if !defined(PLATFORM_PAGE_SIZE) -#define PLATFORM_PAGE_SIZE 4 * 1024 +#define PLATFORM_PAGE_SIZE 4 * 1024 #endif - template <typename T, size_t PadSize = PLATFORM_CACHE_LINE> - struct TPadded: public T { - char Pad[PadSize - sizeof(T) % PadSize]; + template <typename T, size_t PadSize = PLATFORM_CACHE_LINE> + struct TPadded: public T { + char Pad[PadSize - sizeof(T) % PadSize]; - TPadded() { - static_assert(sizeof(*this) % PadSize == 0, "padding does not work"); - Y_UNUSED(Pad); - } + TPadded() { + static_assert(sizeof(*this) % PadSize == 0, "padding does not work"); + Y_UNUSED(Pad); + } template<typename... Args> TPadded(Args&&... args) @@ -42,280 +42,280 @@ namespace NThreading { static_assert(sizeof(*this) % PadSize == 0, "padding does not work"); Y_UNUSED(Pad); } - }; - - //////////////////////////////////////////////////////////////////////////////// - // Type helpers - - namespace NImpl { - template <typename T> - struct TPodTypeHelper { - template <typename TT> - static void Write(T* ptr, TT&& value) { - *ptr = value; - } - - static T Read(T* ptr) { - return *ptr; - } - - static void Destroy(T* ptr) { - Y_UNUSED(ptr); - } - }; - - template <typename T> - struct TNonPodTypeHelper { - template <typename TT> - static void Write(T* ptr, TT&& value) { - new (ptr) T(std::forward<TT>(value)); - } - - static T Read(T* ptr) { - return std::move(*ptr); - } - - static void Destroy(T* ptr) { - (void)ptr; /* Make MSVC happy. */ - ptr->~T(); - } - }; - - template <typename T> - using TTypeHelper = std::conditional_t< - TTypeTraits<T>::IsPod, - TPodTypeHelper<T>, - TNonPodTypeHelper<T>>; - - } - - //////////////////////////////////////////////////////////////////////////////// - // One producer/one consumer chunked queue. - - template <typename T, size_t ChunkSize = PLATFORM_PAGE_SIZE> - class TOneOneQueue: private TNonCopyable { - using TTypeHelper = NImpl::TTypeHelper<T>; - - struct TChunk; - - struct TChunkHeader { - size_t Count = 0; - TChunk* Next = nullptr; - }; - - struct TChunk: public TChunkHeader { - static constexpr size_t MaxCount = (ChunkSize - sizeof(TChunkHeader)) / sizeof(T); - - char Entries[MaxCount * sizeof(T)]; - - TChunk() { - Y_UNUSED(Entries); // uninitialized - } - - ~TChunk() { - for (size_t i = 0; i < this->Count; ++i) { - TTypeHelper::Destroy(GetPtr(i)); - } - } - - T* GetPtr(size_t i) { - return (T*)Entries + i; - } - }; - - struct TWriterState { - TChunk* Chunk = nullptr; - }; - - struct TReaderState { - TChunk* Chunk = nullptr; - size_t Count = 0; - }; - - private: - TPadded<TWriterState> Writer; - TPadded<TReaderState> Reader; - - public: - using TItem = T; - - TOneOneQueue() { - Writer.Chunk = Reader.Chunk = new TChunk(); - } - - ~TOneOneQueue() { - DeleteChunks(Reader.Chunk); - } - - template <typename TT> - void Enqueue(TT&& value) { - T* ptr = PrepareWrite(); - Y_ASSERT(ptr); - TTypeHelper::Write(ptr, std::forward<TT>(value)); - CompleteWrite(); - } - - bool Dequeue(T& value) { - if (T* ptr = PrepareRead()) { - value = TTypeHelper::Read(ptr); - CompleteRead(); - return true; - } - return false; - } - - bool IsEmpty() { - return !PrepareRead(); - } - - protected: - T* PrepareWrite() { - TChunk* chunk = Writer.Chunk; - Y_ASSERT(chunk && !chunk->Next); - - if (chunk->Count != TChunk::MaxCount) { - return chunk->GetPtr(chunk->Count); - } - - chunk = new TChunk(); - AtomicSet(Writer.Chunk->Next, chunk); - Writer.Chunk = chunk; - return chunk->GetPtr(0); - } - - void CompleteWrite() { - AtomicSet(Writer.Chunk->Count, Writer.Chunk->Count + 1); - } - - T* PrepareRead() { - TChunk* chunk = Reader.Chunk; - Y_ASSERT(chunk); - - for (;;) { - size_t writerCount = AtomicGet(chunk->Count); - if (Reader.Count != writerCount) { - return chunk->GetPtr(Reader.Count); - } - - if (writerCount != TChunk::MaxCount) { - return nullptr; - } - - chunk = AtomicGet(chunk->Next); - if (!chunk) { - return nullptr; - } - - delete Reader.Chunk; - Reader.Chunk = chunk; - Reader.Count = 0; - } - } + }; - void CompleteRead() { - ++Reader.Count; - } + //////////////////////////////////////////////////////////////////////////////// + // Type helpers - private: - static void DeleteChunks(TChunk* chunk) { - while (chunk) { - TChunk* next = chunk->Next; - delete chunk; - chunk = next; - } - } - }; + namespace NImpl { + template <typename T> + struct TPodTypeHelper { + template <typename TT> + static void Write(T* ptr, TT&& value) { + *ptr = value; + } - //////////////////////////////////////////////////////////////////////////////// - // Multiple producers/single consumer partitioned queue. - // Provides FIFO guaranties for each producer. + static T Read(T* ptr) { + return *ptr; + } + + static void Destroy(T* ptr) { + Y_UNUSED(ptr); + } + }; - template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> - class TManyOneQueue: private TNonCopyable { - using TTypeHelper = NImpl::TTypeHelper<T>; + template <typename T> + struct TNonPodTypeHelper { + template <typename TT> + static void Write(T* ptr, TT&& value) { + new (ptr) T(std::forward<TT>(value)); + } + + static T Read(T* ptr) { + return std::move(*ptr); + } + + static void Destroy(T* ptr) { + (void)ptr; /* Make MSVC happy. */ + ptr->~T(); + } + }; + + template <typename T> + using TTypeHelper = std::conditional_t< + TTypeTraits<T>::IsPod, + TPodTypeHelper<T>, + TNonPodTypeHelper<T>>; + + } + + //////////////////////////////////////////////////////////////////////////////// + // One producer/one consumer chunked queue. + + template <typename T, size_t ChunkSize = PLATFORM_PAGE_SIZE> + class TOneOneQueue: private TNonCopyable { + using TTypeHelper = NImpl::TTypeHelper<T>; + + struct TChunk; + + struct TChunkHeader { + size_t Count = 0; + TChunk* Next = nullptr; + }; + + struct TChunk: public TChunkHeader { + static constexpr size_t MaxCount = (ChunkSize - sizeof(TChunkHeader)) / sizeof(T); + + char Entries[MaxCount * sizeof(T)]; + + TChunk() { + Y_UNUSED(Entries); // uninitialized + } + + ~TChunk() { + for (size_t i = 0; i < this->Count; ++i) { + TTypeHelper::Destroy(GetPtr(i)); + } + } + + T* GetPtr(size_t i) { + return (T*)Entries + i; + } + }; + + struct TWriterState { + TChunk* Chunk = nullptr; + }; + + struct TReaderState { + TChunk* Chunk = nullptr; + size_t Count = 0; + }; + + private: + TPadded<TWriterState> Writer; + TPadded<TReaderState> Reader; + + public: + using TItem = T; + + TOneOneQueue() { + Writer.Chunk = Reader.Chunk = new TChunk(); + } + + ~TOneOneQueue() { + DeleteChunks(Reader.Chunk); + } + + template <typename TT> + void Enqueue(TT&& value) { + T* ptr = PrepareWrite(); + Y_ASSERT(ptr); + TTypeHelper::Write(ptr, std::forward<TT>(value)); + CompleteWrite(); + } + + bool Dequeue(T& value) { + if (T* ptr = PrepareRead()) { + value = TTypeHelper::Read(ptr); + CompleteRead(); + return true; + } + return false; + } + + bool IsEmpty() { + return !PrepareRead(); + } - struct TEntry { - T Value; - ui64 Tag; - }; - - struct TQueueType: public TOneOneQueue<TEntry, ChunkSize> { - TAtomic WriteLock = 0; + protected: + T* PrepareWrite() { + TChunk* chunk = Writer.Chunk; + Y_ASSERT(chunk && !chunk->Next); + + if (chunk->Count != TChunk::MaxCount) { + return chunk->GetPtr(chunk->Count); + } + + chunk = new TChunk(); + AtomicSet(Writer.Chunk->Next, chunk); + Writer.Chunk = chunk; + return chunk->GetPtr(0); + } + + void CompleteWrite() { + AtomicSet(Writer.Chunk->Count, Writer.Chunk->Count + 1); + } - using TOneOneQueue<TEntry, ChunkSize>::PrepareWrite; - using TOneOneQueue<TEntry, ChunkSize>::CompleteWrite; - - using TOneOneQueue<TEntry, ChunkSize>::PrepareRead; - using TOneOneQueue<TEntry, ChunkSize>::CompleteRead; - }; + T* PrepareRead() { + TChunk* chunk = Reader.Chunk; + Y_ASSERT(chunk); - private: - union { - TAtomic WriteTag = 0; - char Pad[PLATFORM_CACHE_LINE]; - }; - - TQueueType Queues[Concurrency]; - - public: - using TItem = T; - - template <typename TT> - void Enqueue(TT&& value) { - ui64 tag = NextTag(); - while (!TryEnqueue(std::forward<TT>(value), tag)) { - SpinLockPause(); - } - } - - bool Dequeue(T& value) { - size_t index = 0; - if (TEntry* entry = PrepareRead(index)) { - value = TTypeHelper::Read(&entry->Value); - Queues[index].CompleteRead(); - return true; - } - return false; - } + for (;;) { + size_t writerCount = AtomicGet(chunk->Count); + if (Reader.Count != writerCount) { + return chunk->GetPtr(Reader.Count); + } + + if (writerCount != TChunk::MaxCount) { + return nullptr; + } + + chunk = AtomicGet(chunk->Next); + if (!chunk) { + return nullptr; + } + + delete Reader.Chunk; + Reader.Chunk = chunk; + Reader.Count = 0; + } + } + + void CompleteRead() { + ++Reader.Count; + } + + private: + static void DeleteChunks(TChunk* chunk) { + while (chunk) { + TChunk* next = chunk->Next; + delete chunk; + chunk = next; + } + } + }; + + //////////////////////////////////////////////////////////////////////////////// + // Multiple producers/single consumer partitioned queue. + // Provides FIFO guaranties for each producer. + + template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> + class TManyOneQueue: private TNonCopyable { + using TTypeHelper = NImpl::TTypeHelper<T>; + + struct TEntry { + T Value; + ui64 Tag; + }; + + struct TQueueType: public TOneOneQueue<TEntry, ChunkSize> { + TAtomic WriteLock = 0; + + using TOneOneQueue<TEntry, ChunkSize>::PrepareWrite; + using TOneOneQueue<TEntry, ChunkSize>::CompleteWrite; + + using TOneOneQueue<TEntry, ChunkSize>::PrepareRead; + using TOneOneQueue<TEntry, ChunkSize>::CompleteRead; + }; + + private: + union { + TAtomic WriteTag = 0; + char Pad[PLATFORM_CACHE_LINE]; + }; + + TQueueType Queues[Concurrency]; + + public: + using TItem = T; + + template <typename TT> + void Enqueue(TT&& value) { + ui64 tag = NextTag(); + while (!TryEnqueue(std::forward<TT>(value), tag)) { + SpinLockPause(); + } + } + + bool Dequeue(T& value) { + size_t index = 0; + if (TEntry* entry = PrepareRead(index)) { + value = TTypeHelper::Read(&entry->Value); + Queues[index].CompleteRead(); + return true; + } + return false; + } - bool IsEmpty() { - for (size_t i = 0; i < Concurrency; ++i) { - if (!Queues[i].IsEmpty()) { - return false; - } - } - return true; + bool IsEmpty() { + for (size_t i = 0; i < Concurrency; ++i) { + if (!Queues[i].IsEmpty()) { + return false; + } + } + return true; } - private: - ui64 NextTag() { - // TODO: can we avoid synchronization here? it costs 1.5x performance penalty - // return GetCycleCount(); - return AtomicIncrement(WriteTag); - } + private: + ui64 NextTag() { + // TODO: can we avoid synchronization here? it costs 1.5x performance penalty + // return GetCycleCount(); + return AtomicIncrement(WriteTag); + } - template <typename TT> - bool TryEnqueue(TT&& value, ui64 tag) { - for (size_t i = 0; i < Concurrency; ++i) { - TQueueType& queue = Queues[i]; - if (AtomicTryAndTryLock(&queue.WriteLock)) { - TEntry* entry = queue.PrepareWrite(); - Y_ASSERT(entry); - TTypeHelper::Write(&entry->Value, std::forward<TT>(value)); - entry->Tag = tag; - queue.CompleteWrite(); - AtomicUnlock(&queue.WriteLock); - return true; - } + template <typename TT> + bool TryEnqueue(TT&& value, ui64 tag) { + for (size_t i = 0; i < Concurrency; ++i) { + TQueueType& queue = Queues[i]; + if (AtomicTryAndTryLock(&queue.WriteLock)) { + TEntry* entry = queue.PrepareWrite(); + Y_ASSERT(entry); + TTypeHelper::Write(&entry->Value, std::forward<TT>(value)); + entry->Tag = tag; + queue.CompleteWrite(); + AtomicUnlock(&queue.WriteLock); + return true; + } } - return false; + return false; } - TEntry* PrepareRead(size_t& index) { - TEntry* entry = nullptr; - ui64 tag = Max(); + TEntry* PrepareRead(size_t& index) { + TEntry* entry = nullptr; + ui64 tag = Max(); - for (size_t i = 0; i < Concurrency; ++i) { + for (size_t i = 0; i < Concurrency; ++i) { TEntry* e = Queues[i].PrepareRead(); if (e && e->Tag < tag) { index = i; @@ -323,246 +323,246 @@ namespace NThreading { tag = e->Tag; } } - - if (entry) { - // need second pass to catch updates within already scanned range - size_t candidate = index; - for (size_t i = 0; i < candidate; ++i) { - TEntry* e = Queues[i].PrepareRead(); - if (e && e->Tag < tag) { - index = i; - entry = e; - tag = e->Tag; - } - } - } - - return entry; + + if (entry) { + // need second pass to catch updates within already scanned range + size_t candidate = index; + for (size_t i = 0; i < candidate; ++i) { + TEntry* e = Queues[i].PrepareRead(); + if (e && e->Tag < tag) { + index = i; + entry = e; + tag = e->Tag; + } + } + } + + return entry; } - }; + }; - //////////////////////////////////////////////////////////////////////////////// - // Concurrent many-many queue with strong FIFO guaranties. - // Writers will not block readers (and vice versa), but will block each other. + //////////////////////////////////////////////////////////////////////////////// + // Concurrent many-many queue with strong FIFO guaranties. + // Writers will not block readers (and vice versa), but will block each other. - template <typename T, size_t ChunkSize = PLATFORM_PAGE_SIZE, typename TLock = TAdaptiveLock> - class TManyManyQueue: private TNonCopyable { - private: - TPadded<TLock> WriteLock; - TPadded<TLock> ReadLock; - - TOneOneQueue<T, ChunkSize> Queue; - - public: - using TItem = T; - - template <typename TT> - void Enqueue(TT&& value) { - with_lock (WriteLock) { - Queue.Enqueue(std::forward<TT>(value)); - } + template <typename T, size_t ChunkSize = PLATFORM_PAGE_SIZE, typename TLock = TAdaptiveLock> + class TManyManyQueue: private TNonCopyable { + private: + TPadded<TLock> WriteLock; + TPadded<TLock> ReadLock; + + TOneOneQueue<T, ChunkSize> Queue; + + public: + using TItem = T; + + template <typename TT> + void Enqueue(TT&& value) { + with_lock (WriteLock) { + Queue.Enqueue(std::forward<TT>(value)); + } + } + + bool Dequeue(T& value) { + with_lock (ReadLock) { + return Queue.Dequeue(value); + } + } + + bool IsEmpty() { + with_lock (ReadLock) { + return Queue.IsEmpty(); + } + } + }; + + //////////////////////////////////////////////////////////////////////////////// + // Multiple producers/single consumer partitioned queue. + // Because of random partitioning reordering possible - FIFO not guaranteed! + + template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> + class TRelaxedManyOneQueue: private TNonCopyable { + struct TQueueType: public TOneOneQueue<T, ChunkSize> { + TAtomic WriteLock = 0; + }; + + private: + union { + size_t ReadPos = 0; + char Pad[PLATFORM_CACHE_LINE]; + }; + + TQueueType Queues[Concurrency]; + + public: + using TItem = T; + + template <typename TT> + void Enqueue(TT&& value) { + while (!TryEnqueue(std::forward<TT>(value))) { + SpinLockPause(); + } + } + + bool Dequeue(T& value) { + for (size_t i = 0; i < Concurrency; ++i) { + TQueueType& queue = Queues[ReadPos++ % Concurrency]; + if (queue.Dequeue(value)) { + return true; + } + } + return false; + } + + bool IsEmpty() { + for (size_t i = 0; i < Concurrency; ++i) { + if (!Queues[i].IsEmpty()) { + return false; + } + } + return true; + } + + private: + template <typename TT> + bool TryEnqueue(TT&& value) { + size_t writePos = GetCycleCount(); + for (size_t i = 0; i < Concurrency; ++i) { + TQueueType& queue = Queues[writePos++ % Concurrency]; + if (AtomicTryAndTryLock(&queue.WriteLock)) { + queue.Enqueue(std::forward<TT>(value)); + AtomicUnlock(&queue.WriteLock); + return true; + } + } + return false; } + }; - bool Dequeue(T& value) { - with_lock (ReadLock) { - return Queue.Dequeue(value); - } - } - - bool IsEmpty() { - with_lock (ReadLock) { - return Queue.IsEmpty(); - } - } - }; - - //////////////////////////////////////////////////////////////////////////////// - // Multiple producers/single consumer partitioned queue. - // Because of random partitioning reordering possible - FIFO not guaranteed! - - template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> - class TRelaxedManyOneQueue: private TNonCopyable { - struct TQueueType: public TOneOneQueue<T, ChunkSize> { - TAtomic WriteLock = 0; - }; - - private: - union { - size_t ReadPos = 0; - char Pad[PLATFORM_CACHE_LINE]; + //////////////////////////////////////////////////////////////////////////////// + // Concurrent many-many partitioned queue. + // Because of random partitioning reordering possible - FIFO not guaranteed! + + template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> + class TRelaxedManyManyQueue: private TNonCopyable { + struct TQueueType: public TOneOneQueue<T, ChunkSize> { + union { + TAtomic WriteLock = 0; + char Pad1[PLATFORM_CACHE_LINE]; + }; + union { + TAtomic ReadLock = 0; + char Pad2[PLATFORM_CACHE_LINE]; + }; }; - TQueueType Queues[Concurrency]; - - public: - using TItem = T; - - template <typename TT> - void Enqueue(TT&& value) { - while (!TryEnqueue(std::forward<TT>(value))) { - SpinLockPause(); - } - } - - bool Dequeue(T& value) { - for (size_t i = 0; i < Concurrency; ++i) { - TQueueType& queue = Queues[ReadPos++ % Concurrency]; - if (queue.Dequeue(value)) { - return true; + private: + TQueueType Queues[Concurrency]; + + public: + using TItem = T; + + template <typename TT> + void Enqueue(TT&& value) { + while (!TryEnqueue(std::forward<TT>(value))) { + SpinLockPause(); + } + } + + bool Dequeue(T& value) { + size_t readPos = GetCycleCount(); + for (size_t i = 0; i < Concurrency; ++i) { + TQueueType& queue = Queues[readPos++ % Concurrency]; + if (AtomicTryAndTryLock(&queue.ReadLock)) { + bool dequeued = queue.Dequeue(value); + AtomicUnlock(&queue.ReadLock); + if (dequeued) { + return true; + } } } - return false; + return false; } - bool IsEmpty() { - for (size_t i = 0; i < Concurrency; ++i) { - if (!Queues[i].IsEmpty()) { - return false; + bool IsEmpty() { + for (size_t i = 0; i < Concurrency; ++i) { + TQueueType& queue = Queues[i]; + if (AtomicTryAndTryLock(&queue.ReadLock)) { + bool empty = queue.IsEmpty(); + AtomicUnlock(&queue.ReadLock); + if (!empty) { + return false; + } } } - return true; + return true; } - private: - template <typename TT> - bool TryEnqueue(TT&& value) { - size_t writePos = GetCycleCount(); - for (size_t i = 0; i < Concurrency; ++i) { - TQueueType& queue = Queues[writePos++ % Concurrency]; - if (AtomicTryAndTryLock(&queue.WriteLock)) { - queue.Enqueue(std::forward<TT>(value)); - AtomicUnlock(&queue.WriteLock); - return true; - } + private: + template <typename TT> + bool TryEnqueue(TT&& value) { + size_t writePos = GetCycleCount(); + for (size_t i = 0; i < Concurrency; ++i) { + TQueueType& queue = Queues[writePos++ % Concurrency]; + if (AtomicTryAndTryLock(&queue.WriteLock)) { + queue.Enqueue(std::forward<TT>(value)); + AtomicUnlock(&queue.WriteLock); + return true; + } } - return false; + return false; } - }; - - //////////////////////////////////////////////////////////////////////////////// - // Concurrent many-many partitioned queue. - // Because of random partitioning reordering possible - FIFO not guaranteed! - - template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> - class TRelaxedManyManyQueue: private TNonCopyable { - struct TQueueType: public TOneOneQueue<T, ChunkSize> { - union { - TAtomic WriteLock = 0; - char Pad1[PLATFORM_CACHE_LINE]; - }; - union { - TAtomic ReadLock = 0; - char Pad2[PLATFORM_CACHE_LINE]; - }; - }; + }; - private: - TQueueType Queues[Concurrency]; + //////////////////////////////////////////////////////////////////////////////// + // Simple wrapper to deal with AutoPtrs - public: - using TItem = T; + template <typename T, typename TImpl> + class TAutoQueueBase: private TNonCopyable { + private: + TImpl Impl; - template <typename TT> - void Enqueue(TT&& value) { - while (!TryEnqueue(std::forward<TT>(value))) { - SpinLockPause(); - } - } + public: + using TItem = TAutoPtr<T>; - bool Dequeue(T& value) { - size_t readPos = GetCycleCount(); - for (size_t i = 0; i < Concurrency; ++i) { - TQueueType& queue = Queues[readPos++ % Concurrency]; - if (AtomicTryAndTryLock(&queue.ReadLock)) { - bool dequeued = queue.Dequeue(value); - AtomicUnlock(&queue.ReadLock); - if (dequeued) { - return true; - } - } - } - return false; + ~TAutoQueueBase() { + TAutoPtr<T> value; + while (Dequeue(value)) { + // do nothing + } } - bool IsEmpty() { - for (size_t i = 0; i < Concurrency; ++i) { - TQueueType& queue = Queues[i]; - if (AtomicTryAndTryLock(&queue.ReadLock)) { - bool empty = queue.IsEmpty(); - AtomicUnlock(&queue.ReadLock); - if (!empty) { - return false; - } - } - } - return true; - } - - private: - template <typename TT> - bool TryEnqueue(TT&& value) { - size_t writePos = GetCycleCount(); - for (size_t i = 0; i < Concurrency; ++i) { - TQueueType& queue = Queues[writePos++ % Concurrency]; - if (AtomicTryAndTryLock(&queue.WriteLock)) { - queue.Enqueue(std::forward<TT>(value)); - AtomicUnlock(&queue.WriteLock); - return true; - } - } - return false; - } - }; - - //////////////////////////////////////////////////////////////////////////////// - // Simple wrapper to deal with AutoPtrs - - template <typename T, typename TImpl> - class TAutoQueueBase: private TNonCopyable { - private: - TImpl Impl; - - public: - using TItem = TAutoPtr<T>; - - ~TAutoQueueBase() { - TAutoPtr<T> value; - while (Dequeue(value)) { - // do nothing - } - } - - void Enqueue(TAutoPtr<T> value) { - Impl.Enqueue(value.Get()); + void Enqueue(TAutoPtr<T> value) { + Impl.Enqueue(value.Get()); Y_UNUSED(value.Release()); - } + } - bool Dequeue(TAutoPtr<T>& value) { - T* ptr = nullptr; - if (Impl.Dequeue(ptr)) { - value.Reset(ptr); - return true; - } - return false; + bool Dequeue(TAutoPtr<T>& value) { + T* ptr = nullptr; + if (Impl.Dequeue(ptr)) { + value.Reset(ptr); + return true; + } + return false; } - bool IsEmpty() { - return Impl.IsEmpty(); - } - }; + bool IsEmpty() { + return Impl.IsEmpty(); + } + }; - template <typename T, size_t ChunkSize = PLATFORM_PAGE_SIZE> - using TAutoOneOneQueue = TAutoQueueBase<T, TOneOneQueue<T*, ChunkSize>>; + template <typename T, size_t ChunkSize = PLATFORM_PAGE_SIZE> + using TAutoOneOneQueue = TAutoQueueBase<T, TOneOneQueue<T*, ChunkSize>>; - template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> - using TAutoManyOneQueue = TAutoQueueBase<T, TManyOneQueue<T*, Concurrency, ChunkSize>>; + template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> + using TAutoManyOneQueue = TAutoQueueBase<T, TManyOneQueue<T*, Concurrency, ChunkSize>>; - template <typename T, size_t ChunkSize = PLATFORM_PAGE_SIZE, typename TLock = TAdaptiveLock> - using TAutoManyManyQueue = TAutoQueueBase<T, TManyManyQueue<T*, ChunkSize, TLock>>; + template <typename T, size_t ChunkSize = PLATFORM_PAGE_SIZE, typename TLock = TAdaptiveLock> + using TAutoManyManyQueue = TAutoQueueBase<T, TManyManyQueue<T*, ChunkSize, TLock>>; - template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> - using TAutoRelaxedManyOneQueue = TAutoQueueBase<T, TRelaxedManyOneQueue<T*, Concurrency, ChunkSize>>; + template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> + using TAutoRelaxedManyOneQueue = TAutoQueueBase<T, TRelaxedManyOneQueue<T*, Concurrency, ChunkSize>>; - template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> - using TAutoRelaxedManyManyQueue = TAutoQueueBase<T, TRelaxedManyManyQueue<T*, Concurrency, ChunkSize>>; -} + template <typename T, size_t Concurrency = 4, size_t ChunkSize = PLATFORM_PAGE_SIZE> + using TAutoRelaxedManyManyQueue = TAutoQueueBase<T, TRelaxedManyManyQueue<T*, Concurrency, ChunkSize>>; +} diff --git a/library/cpp/threading/chunk_queue/queue_ut.cpp b/library/cpp/threading/chunk_queue/queue_ut.cpp index 8cb36d8dd1..406b71dd4c 100644 --- a/library/cpp/threading/chunk_queue/queue_ut.cpp +++ b/library/cpp/threading/chunk_queue/queue_ut.cpp @@ -5,55 +5,55 @@ #include <util/generic/set.h> namespace NThreading { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// Y_UNIT_TEST_SUITE(TOneOneQueueTest){ Y_UNIT_TEST(ShouldBeEmptyAtStart){ - TOneOneQueue<int> queue; + TOneOneQueue<int> queue; - int result = 0; - UNIT_ASSERT(queue.IsEmpty()); - UNIT_ASSERT(!queue.Dequeue(result)); -} + int result = 0; + UNIT_ASSERT(queue.IsEmpty()); + UNIT_ASSERT(!queue.Dequeue(result)); +} Y_UNIT_TEST(ShouldReturnEntries) { - TOneOneQueue<int> queue; - queue.Enqueue(1); - queue.Enqueue(2); - queue.Enqueue(3); + TOneOneQueue<int> queue; + queue.Enqueue(1); + queue.Enqueue(2); + queue.Enqueue(3); - int result = 0; - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT_EQUAL(result, 1); + int result = 0; + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT_EQUAL(result, 1); - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT_EQUAL(result, 2); + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT_EQUAL(result, 2); - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT_EQUAL(result, 3); + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT_EQUAL(result, 3); - UNIT_ASSERT(queue.IsEmpty()); - UNIT_ASSERT(!queue.Dequeue(result)); -} + UNIT_ASSERT(queue.IsEmpty()); + UNIT_ASSERT(!queue.Dequeue(result)); +} Y_UNIT_TEST(ShouldStoreMultipleChunks) { - TOneOneQueue<int, 100> queue; - for (int i = 0; i < 1000; ++i) { - queue.Enqueue(i); + TOneOneQueue<int, 100> queue; + for (int i = 0; i < 1000; ++i) { + queue.Enqueue(i); } - for (int i = 0; i < 1000; ++i) { - int result = 0; - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT_EQUAL(result, i); + for (int i = 0; i < 1000; ++i) { + int result = 0; + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT_EQUAL(result, i); } -} -} -; +} +} +; //////////////////////////////////////////////////////////////////////////////// @@ -61,35 +61,35 @@ Y_UNIT_TEST_SUITE(TManyOneQueueTest){ Y_UNIT_TEST(ShouldBeEmptyAtStart){ TManyOneQueue<int> queue; -int result; -UNIT_ASSERT(queue.IsEmpty()); -UNIT_ASSERT(!queue.Dequeue(result)); -} +int result; +UNIT_ASSERT(queue.IsEmpty()); +UNIT_ASSERT(!queue.Dequeue(result)); +} Y_UNIT_TEST(ShouldReturnEntries) { - TManyOneQueue<int> queue; - queue.Enqueue(1); - queue.Enqueue(2); - queue.Enqueue(3); - - int result = 0; - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT_EQUAL(result, 1); - - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT_EQUAL(result, 2); - - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT_EQUAL(result, 3); - - UNIT_ASSERT(queue.IsEmpty()); - UNIT_ASSERT(!queue.Dequeue(result)); -} -} -; + TManyOneQueue<int> queue; + queue.Enqueue(1); + queue.Enqueue(2); + queue.Enqueue(3); + + int result = 0; + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT_EQUAL(result, 1); + + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT_EQUAL(result, 2); + + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT_EQUAL(result, 3); + + UNIT_ASSERT(queue.IsEmpty()); + UNIT_ASSERT(!queue.Dequeue(result)); +} +} +; //////////////////////////////////////////////////////////////////////////////// @@ -97,35 +97,35 @@ Y_UNIT_TEST_SUITE(TManyManyQueueTest){ Y_UNIT_TEST(ShouldBeEmptyAtStart){ TManyManyQueue<int> queue; -int result = 0; -UNIT_ASSERT(queue.IsEmpty()); -UNIT_ASSERT(!queue.Dequeue(result)); -} +int result = 0; +UNIT_ASSERT(queue.IsEmpty()); +UNIT_ASSERT(!queue.Dequeue(result)); +} Y_UNIT_TEST(ShouldReturnEntries) { - TManyManyQueue<int> queue; - queue.Enqueue(1); - queue.Enqueue(2); - queue.Enqueue(3); - - int result = 0; - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT_EQUAL(result, 1); - - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT_EQUAL(result, 2); - - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT_EQUAL(result, 3); - - UNIT_ASSERT(queue.IsEmpty()); - UNIT_ASSERT(!queue.Dequeue(result)); -} -} -; + TManyManyQueue<int> queue; + queue.Enqueue(1); + queue.Enqueue(2); + queue.Enqueue(3); + + int result = 0; + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT_EQUAL(result, 1); + + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT_EQUAL(result, 2); + + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT_EQUAL(result, 3); + + UNIT_ASSERT(queue.IsEmpty()); + UNIT_ASSERT(!queue.Dequeue(result)); +} +} +; //////////////////////////////////////////////////////////////////////////////// @@ -133,37 +133,37 @@ Y_UNIT_TEST_SUITE(TRelaxedManyOneQueueTest){ Y_UNIT_TEST(ShouldBeEmptyAtStart){ TRelaxedManyOneQueue<int> queue; -int result; -UNIT_ASSERT(queue.IsEmpty()); -UNIT_ASSERT(!queue.Dequeue(result)); -} +int result; +UNIT_ASSERT(queue.IsEmpty()); +UNIT_ASSERT(!queue.Dequeue(result)); +} Y_UNIT_TEST(ShouldReturnEntries) { - TSet<int> items = {1, 2, 3}; + TSet<int> items = {1, 2, 3}; - TRelaxedManyOneQueue<int> queue; - for (int item : items) { - queue.Enqueue(item); - } + TRelaxedManyOneQueue<int> queue; + for (int item : items) { + queue.Enqueue(item); + } - int result = 0; - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT(items.erase(result)); + int result = 0; + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT(items.erase(result)); - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT(items.erase(result)); + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT(items.erase(result)); - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT(items.erase(result)); + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT(items.erase(result)); - UNIT_ASSERT(queue.IsEmpty()); - UNIT_ASSERT(!queue.Dequeue(result)); -} -} -; + UNIT_ASSERT(queue.IsEmpty()); + UNIT_ASSERT(!queue.Dequeue(result)); +} +} +; //////////////////////////////////////////////////////////////////////////////// @@ -171,35 +171,35 @@ Y_UNIT_TEST_SUITE(TRelaxedManyManyQueueTest){ Y_UNIT_TEST(ShouldBeEmptyAtStart){ TRelaxedManyManyQueue<int> queue; -int result = 0; -UNIT_ASSERT(queue.IsEmpty()); -UNIT_ASSERT(!queue.Dequeue(result)); -} +int result = 0; +UNIT_ASSERT(queue.IsEmpty()); +UNIT_ASSERT(!queue.Dequeue(result)); +} Y_UNIT_TEST(ShouldReturnEntries) { - TSet<int> items = {1, 2, 3}; - - TRelaxedManyManyQueue<int> queue; - for (int item : items) { - queue.Enqueue(item); - } - - int result = 0; - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT(items.erase(result)); - - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT(items.erase(result)); - - UNIT_ASSERT(!queue.IsEmpty()); - UNIT_ASSERT(queue.Dequeue(result)); - UNIT_ASSERT(items.erase(result)); - - UNIT_ASSERT(queue.IsEmpty()); - UNIT_ASSERT(!queue.Dequeue(result)); -} -} -; -} + TSet<int> items = {1, 2, 3}; + + TRelaxedManyManyQueue<int> queue; + for (int item : items) { + queue.Enqueue(item); + } + + int result = 0; + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT(items.erase(result)); + + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT(items.erase(result)); + + UNIT_ASSERT(!queue.IsEmpty()); + UNIT_ASSERT(queue.Dequeue(result)); + UNIT_ASSERT(items.erase(result)); + + UNIT_ASSERT(queue.IsEmpty()); + UNIT_ASSERT(!queue.Dequeue(result)); +} +} +; +} diff --git a/library/cpp/threading/future/async.h b/library/cpp/threading/future/async.h index 8543fdd5c6..f964d2dc88 100644 --- a/library/cpp/threading/future/async.h +++ b/library/cpp/threading/future/async.h @@ -2,11 +2,11 @@ #include "future.h" -#include <util/generic/function.h> +#include <util/generic/function.h> #include <util/thread/pool.h> namespace NThreading { - /** + /** * @brief Asynchronously executes @arg func in @arg queue returning a future for the result. * * @arg func should be a callable object with signature T(). @@ -17,15 +17,15 @@ namespace NThreading { * If you want to use another queue for execution just write an overload, @see ExtensionExample * unittest. */ - template <typename Func> + template <typename Func> TFuture<TFutureType<TFunctionResult<Func>>> Async(Func&& func, IThreadPool& queue) { - auto promise = NewPromise<TFutureType<TFunctionResult<Func>>>(); + auto promise = NewPromise<TFutureType<TFunctionResult<Func>>>(); auto lambda = [promise, func = std::forward<Func>(func)]() mutable { - NImpl::SetValue(promise, func); + NImpl::SetValue(promise, func); }; queue.SafeAddFunc(std::move(lambda)); - return promise.GetFuture(); - } + return promise.GetFuture(); + } } diff --git a/library/cpp/threading/future/async_ut.cpp b/library/cpp/threading/future/async_ut.cpp index a3699744e4..a452965dbc 100644 --- a/library/cpp/threading/future/async_ut.cpp +++ b/library/cpp/threading/future/async_ut.cpp @@ -6,13 +6,13 @@ #include <util/generic/vector.h> namespace { - struct TMySuperTaskQueue { - }; + struct TMySuperTaskQueue { + }; } namespace NThreading { - /* Here we provide an Async overload for TMySuperTaskQueue indide NThreading namespace + /* Here we provide an Async overload for TMySuperTaskQueue indide NThreading namespace * so that we can call it in the way * * TMySuperTaskQueue queue; @@ -20,38 +20,38 @@ namespace NThreading { * * See also ExtensionExample unittest. */ - template <typename Func> - TFuture<TFunctionResult<Func>> Async(Func func, TMySuperTaskQueue&) { - return MakeFuture(func()); - } + template <typename Func> + TFuture<TFunctionResult<Func>> Async(Func func, TMySuperTaskQueue&) { + return MakeFuture(func()); + } } Y_UNIT_TEST_SUITE(Async) { Y_UNIT_TEST(ExtensionExample) { - TMySuperTaskQueue queue; - auto future = NThreading::Async([]() { return 5; }, queue); - future.Wait(); - UNIT_ASSERT_VALUES_EQUAL(future.GetValue(), 5); - } + TMySuperTaskQueue queue; + auto future = NThreading::Async([]() { return 5; }, queue); + future.Wait(); + UNIT_ASSERT_VALUES_EQUAL(future.GetValue(), 5); + } Y_UNIT_TEST(WorksWithIMtpQueue) { auto queue = MakeHolder<TThreadPool>(); - queue->Start(1); + queue->Start(1); - auto future = NThreading::Async([]() { return 5; }, *queue); - future.Wait(); - UNIT_ASSERT_VALUES_EQUAL(future.GetValue(), 5); - } + auto future = NThreading::Async([]() { return 5; }, *queue); + future.Wait(); + UNIT_ASSERT_VALUES_EQUAL(future.GetValue(), 5); + } Y_UNIT_TEST(ProperlyDeducesFutureType) { - // Compileability test + // Compileability test auto queue = CreateThreadPool(1); - NThreading::TFuture<void> f1 = NThreading::Async([]() {}, *queue); - NThreading::TFuture<int> f2 = NThreading::Async([]() { return 5; }, *queue); - NThreading::TFuture<double> f3 = NThreading::Async([]() { return 5.0; }, *queue); - NThreading::TFuture<TVector<int>> f4 = NThreading::Async([]() { return TVector<int>(); }, *queue); - NThreading::TFuture<int> f5 = NThreading::Async([]() { return NThreading::MakeFuture(5); }, *queue); - } + NThreading::TFuture<void> f1 = NThreading::Async([]() {}, *queue); + NThreading::TFuture<int> f2 = NThreading::Async([]() { return 5; }, *queue); + NThreading::TFuture<double> f3 = NThreading::Async([]() { return 5.0; }, *queue); + NThreading::TFuture<TVector<int>> f4 = NThreading::Async([]() { return TVector<int>(); }, *queue); + NThreading::TFuture<int> f5 = NThreading::Async([]() { return NThreading::MakeFuture(5); }, *queue); + } } diff --git a/library/cpp/threading/future/core/future-inl.h b/library/cpp/threading/future/core/future-inl.h index 5fd4296a93..a0e06c1891 100644 --- a/library/cpp/threading/future/core/future-inl.h +++ b/library/cpp/threading/future/core/future-inl.h @@ -2,92 +2,92 @@ #if !defined(INCLUDE_FUTURE_INL_H) #error "you should never include future-inl.h directly" -#endif // INCLUDE_FUTURE_INL_H +#endif // INCLUDE_FUTURE_INL_H namespace NThreading { - namespace NImpl { - //////////////////////////////////////////////////////////////////////////////// + namespace NImpl { + //////////////////////////////////////////////////////////////////////////////// - template <typename T> - using TCallback = std::function<void(const TFuture<T>&)>; + template <typename T> + using TCallback = std::function<void(const TFuture<T>&)>; - template <typename T> - using TCallbackList = TVector<TCallback<T>>; // TODO: small vector + template <typename T> + using TCallbackList = TVector<TCallback<T>>; // TODO: small vector - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// enum class TError { Error }; - template <typename T> - class TFutureState: public TAtomicRefCount<TFutureState<T>> { - enum { - NotReady, - ExceptionSet, - ValueMoved, // keep the ordering of this and following values - ValueSet, - ValueRead, - }; - - private: - mutable TAtomic State; - TAdaptiveLock StateLock; - - TCallbackList<T> Callbacks; + template <typename T> + class TFutureState: public TAtomicRefCount<TFutureState<T>> { + enum { + NotReady, + ExceptionSet, + ValueMoved, // keep the ordering of this and following values + ValueSet, + ValueRead, + }; + + private: + mutable TAtomic State; + TAdaptiveLock StateLock; + + TCallbackList<T> Callbacks; mutable THolder<TSystemEvent> ReadyEvent; - std::exception_ptr Exception; - - union { - char NullValue; - T Value; - }; - - void AccessValue(TDuration timeout, int acquireState) const { - int state = AtomicGet(State); - if (Y_UNLIKELY(state == NotReady)) { - if (timeout == TDuration::Zero()) { - ythrow TFutureException() << "value not set"; - } - - if (!Wait(timeout)) { - ythrow TFutureException() << "wait timeout"; - } - - state = AtomicGet(State); - } - + std::exception_ptr Exception; + + union { + char NullValue; + T Value; + }; + + void AccessValue(TDuration timeout, int acquireState) const { + int state = AtomicGet(State); + if (Y_UNLIKELY(state == NotReady)) { + if (timeout == TDuration::Zero()) { + ythrow TFutureException() << "value not set"; + } + + if (!Wait(timeout)) { + ythrow TFutureException() << "wait timeout"; + } + + state = AtomicGet(State); + } + TryRethrowWithState(state); - - switch (AtomicGetAndCas(&State, acquireState, ValueSet)) { - case ValueSet: - break; - case ValueRead: - if (acquireState != ValueRead) { - ythrow TFutureException() << "value being read"; - } - break; - case ValueMoved: - ythrow TFutureException() << "value was moved"; - default: - Y_ASSERT(state == ValueSet); - } - } - - public: - TFutureState() - : State(NotReady) - , NullValue(0) - { - } - - template <typename TT> - TFutureState(TT&& value) - : State(ValueSet) - , Value(std::forward<TT>(value)) - { - } + + switch (AtomicGetAndCas(&State, acquireState, ValueSet)) { + case ValueSet: + break; + case ValueRead: + if (acquireState != ValueRead) { + ythrow TFutureException() << "value being read"; + } + break; + case ValueMoved: + ythrow TFutureException() << "value was moved"; + default: + Y_ASSERT(state == ValueSet); + } + } + + public: + TFutureState() + : State(NotReady) + , NullValue(0) + { + } + + template <typename TT> + TFutureState(TT&& value) + : State(ValueSet) + , Value(std::forward<TT>(value)) + { + } TFutureState(std::exception_ptr exception, TError) : State(ExceptionSet) @@ -96,14 +96,14 @@ namespace NThreading { { } - ~TFutureState() { - if (State >= ValueMoved) { // ValueMoved, ValueSet, ValueRead - Value.~T(); - } - } + ~TFutureState() { + if (State >= ValueMoved) { // ValueMoved, ValueSet, ValueRead + Value.~T(); + } + } - bool HasValue() const { - return AtomicGet(State) >= ValueMoved; // ValueMoved, ValueSet, ValueRead + bool HasValue() const { + return AtomicGet(State) >= ValueMoved; // ValueMoved, ValueSet, ValueRead } void TryRethrow() const { @@ -111,22 +111,22 @@ namespace NThreading { TryRethrowWithState(state); } - bool HasException() const { - return AtomicGet(State) == ExceptionSet; - } + bool HasException() const { + return AtomicGet(State) == ExceptionSet; + } - const T& GetValue(TDuration timeout = TDuration::Zero()) const { - AccessValue(timeout, ValueRead); - return Value; - } + const T& GetValue(TDuration timeout = TDuration::Zero()) const { + AccessValue(timeout, ValueRead); + return Value; + } - T ExtractValue(TDuration timeout = TDuration::Zero()) { - AccessValue(timeout, ValueMoved); - return std::move(Value); - } + T ExtractValue(TDuration timeout = TDuration::Zero()) { + AccessValue(timeout, ValueMoved); + return std::move(Value); + } - template <typename TT> - void SetValue(TT&& value) { + template <typename TT> + void SetValue(TT&& value) { bool success = TrySetValue(std::forward<TT>(value)); if (Y_UNLIKELY(!success)) { ythrow TFutureException() << "value already set"; @@ -136,37 +136,37 @@ namespace NThreading { template <typename TT> bool TrySetValue(TT&& value) { TSystemEvent* readyEvent = nullptr; - TCallbackList<T> callbacks; + TCallbackList<T> callbacks; - with_lock (StateLock) { - int state = AtomicGet(State); - if (Y_UNLIKELY(state != NotReady)) { + with_lock (StateLock) { + int state = AtomicGet(State); + if (Y_UNLIKELY(state != NotReady)) { return false; - } - - new (&Value) T(std::forward<TT>(value)); + } - readyEvent = ReadyEvent.Get(); - callbacks = std::move(Callbacks); + new (&Value) T(std::forward<TT>(value)); - AtomicSet(State, ValueSet); - } + readyEvent = ReadyEvent.Get(); + callbacks = std::move(Callbacks); - if (readyEvent) { - readyEvent->Signal(); - } + AtomicSet(State, ValueSet); + } - if (callbacks) { - TFuture<T> temp(this); - for (auto& callback : callbacks) { - callback(temp); - } - } + if (readyEvent) { + readyEvent->Signal(); + } + + if (callbacks) { + TFuture<T> temp(this); + for (auto& callback : callbacks) { + callback(temp); + } + } return true; } - void SetException(std::exception_ptr e) { + void SetException(std::exception_ptr e) { bool success = TrySetException(std::move(e)); if (Y_UNLIKELY(!success)) { ythrow TFutureException() << "value already set"; @@ -175,73 +175,73 @@ namespace NThreading { bool TrySetException(std::exception_ptr e) { TSystemEvent* readyEvent; - TCallbackList<T> callbacks; + TCallbackList<T> callbacks; - with_lock (StateLock) { - int state = AtomicGet(State); - if (Y_UNLIKELY(state != NotReady)) { + with_lock (StateLock) { + int state = AtomicGet(State); + if (Y_UNLIKELY(state != NotReady)) { return false; - } - - Exception = std::move(e); - - readyEvent = ReadyEvent.Get(); - callbacks = std::move(Callbacks); - - AtomicSet(State, ExceptionSet); - } - - if (readyEvent) { - readyEvent->Signal(); - } - - if (callbacks) { - TFuture<T> temp(this); - for (auto& callback : callbacks) { - callback(temp); - } - } + } + + Exception = std::move(e); + + readyEvent = ReadyEvent.Get(); + callbacks = std::move(Callbacks); + + AtomicSet(State, ExceptionSet); + } + + if (readyEvent) { + readyEvent->Signal(); + } + + if (callbacks) { + TFuture<T> temp(this); + for (auto& callback : callbacks) { + callback(temp); + } + } return true; } - template <typename F> - bool Subscribe(F&& func) { - with_lock (StateLock) { - int state = AtomicGet(State); - if (state == NotReady) { - Callbacks.emplace_back(std::forward<F>(func)); - return true; - } - } - return false; - } + template <typename F> + bool Subscribe(F&& func) { + with_lock (StateLock) { + int state = AtomicGet(State); + if (state == NotReady) { + Callbacks.emplace_back(std::forward<F>(func)); + return true; + } + } + return false; + } - void Wait() const { - Wait(TInstant::Max()); + void Wait() const { + Wait(TInstant::Max()); } - bool Wait(TDuration timeout) const { - return Wait(timeout.ToDeadLine()); - } + bool Wait(TDuration timeout) const { + return Wait(timeout.ToDeadLine()); + } - bool Wait(TInstant deadline) const { + bool Wait(TInstant deadline) const { TSystemEvent* readyEvent = nullptr; - with_lock (StateLock) { - int state = AtomicGet(State); - if (state != NotReady) { - return true; - } + with_lock (StateLock) { + int state = AtomicGet(State); + if (state != NotReady) { + return true; + } - if (!ReadyEvent) { + if (!ReadyEvent) { ReadyEvent.Reset(new TSystemEvent()); - } - readyEvent = ReadyEvent.Get(); - } + } + readyEvent = ReadyEvent.Get(); + } - Y_ASSERT(readyEvent); - return readyEvent->WaitD(deadline); + Y_ASSERT(readyEvent); + return readyEvent->WaitD(deadline); } void TryRethrowWithState(int state) const { @@ -250,31 +250,31 @@ namespace NThreading { std::rethrow_exception(Exception); } } - }; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - template <> - class TFutureState<void>: public TAtomicRefCount<TFutureState<void>> { - enum { - NotReady, - ValueSet, - ExceptionSet, - }; + template <> + class TFutureState<void>: public TAtomicRefCount<TFutureState<void>> { + enum { + NotReady, + ValueSet, + ExceptionSet, + }; - private: - TAtomic State; - TAdaptiveLock StateLock; + private: + TAtomic State; + TAdaptiveLock StateLock; - TCallbackList<void> Callbacks; + TCallbackList<void> Callbacks; mutable THolder<TSystemEvent> ReadyEvent; - std::exception_ptr Exception; - - public: - TFutureState(bool valueSet = false) - : State(valueSet ? ValueSet : NotReady) - { + std::exception_ptr Exception; + + public: + TFutureState(bool valueSet = false) + : State(valueSet ? ValueSet : NotReady) + { } TFutureState(std::exception_ptr exception, TError) @@ -283,8 +283,8 @@ namespace NThreading { { } - bool HasValue() const { - return AtomicGet(State) == ValueSet; + bool HasValue() const { + return AtomicGet(State) == ValueSet; } void TryRethrow() const { @@ -292,30 +292,30 @@ namespace NThreading { TryRethrowWithState(state); } - bool HasException() const { - return AtomicGet(State) == ExceptionSet; - } + bool HasException() const { + return AtomicGet(State) == ExceptionSet; + } - void GetValue(TDuration timeout = TDuration::Zero()) const { - int state = AtomicGet(State); - if (Y_UNLIKELY(state == NotReady)) { - if (timeout == TDuration::Zero()) { - ythrow TFutureException() << "value not set"; - } + void GetValue(TDuration timeout = TDuration::Zero()) const { + int state = AtomicGet(State); + if (Y_UNLIKELY(state == NotReady)) { + if (timeout == TDuration::Zero()) { + ythrow TFutureException() << "value not set"; + } - if (!Wait(timeout)) { - ythrow TFutureException() << "wait timeout"; - } + if (!Wait(timeout)) { + ythrow TFutureException() << "wait timeout"; + } - state = AtomicGet(State); - } + state = AtomicGet(State); + } TryRethrowWithState(state); - Y_ASSERT(state == ValueSet); - } + Y_ASSERT(state == ValueSet); + } - void SetValue() { + void SetValue() { bool success = TrySetValue(); if (Y_UNLIKELY(!success)) { ythrow TFutureException() << "value already set"; @@ -324,35 +324,35 @@ namespace NThreading { bool TrySetValue() { TSystemEvent* readyEvent = nullptr; - TCallbackList<void> callbacks; + TCallbackList<void> callbacks; - with_lock (StateLock) { - int state = AtomicGet(State); - if (Y_UNLIKELY(state != NotReady)) { + with_lock (StateLock) { + int state = AtomicGet(State); + if (Y_UNLIKELY(state != NotReady)) { return false; - } + } - readyEvent = ReadyEvent.Get(); - callbacks = std::move(Callbacks); + readyEvent = ReadyEvent.Get(); + callbacks = std::move(Callbacks); - AtomicSet(State, ValueSet); - } + AtomicSet(State, ValueSet); + } - if (readyEvent) { - readyEvent->Signal(); - } - - if (callbacks) { - TFuture<void> temp(this); - for (auto& callback : callbacks) { - callback(temp); - } - } + if (readyEvent) { + readyEvent->Signal(); + } + + if (callbacks) { + TFuture<void> temp(this); + for (auto& callback : callbacks) { + callback(temp); + } + } return true; } - void SetException(std::exception_ptr e) { + void SetException(std::exception_ptr e) { bool success = TrySetException(std::move(e)); if (Y_UNLIKELY(!success)) { ythrow TFutureException() << "value already set"; @@ -361,73 +361,73 @@ namespace NThreading { bool TrySetException(std::exception_ptr e) { TSystemEvent* readyEvent = nullptr; - TCallbackList<void> callbacks; + TCallbackList<void> callbacks; - with_lock (StateLock) { - int state = AtomicGet(State); - if (Y_UNLIKELY(state != NotReady)) { + with_lock (StateLock) { + int state = AtomicGet(State); + if (Y_UNLIKELY(state != NotReady)) { return false; - } + } - Exception = std::move(e); + Exception = std::move(e); - readyEvent = ReadyEvent.Get(); - callbacks = std::move(Callbacks); + readyEvent = ReadyEvent.Get(); + callbacks = std::move(Callbacks); - AtomicSet(State, ExceptionSet); - } + AtomicSet(State, ExceptionSet); + } - if (readyEvent) { - readyEvent->Signal(); - } + if (readyEvent) { + readyEvent->Signal(); + } - if (callbacks) { - TFuture<void> temp(this); - for (auto& callback : callbacks) { - callback(temp); - } - } + if (callbacks) { + TFuture<void> temp(this); + for (auto& callback : callbacks) { + callback(temp); + } + } return true; - } + } - template <typename F> - bool Subscribe(F&& func) { - with_lock (StateLock) { - int state = AtomicGet(State); - if (state == NotReady) { - Callbacks.emplace_back(std::forward<F>(func)); - return true; - } - } - return false; - } + template <typename F> + bool Subscribe(F&& func) { + with_lock (StateLock) { + int state = AtomicGet(State); + if (state == NotReady) { + Callbacks.emplace_back(std::forward<F>(func)); + return true; + } + } + return false; + } - void Wait() const { - Wait(TInstant::Max()); + void Wait() const { + Wait(TInstant::Max()); } - bool Wait(TDuration timeout) const { - return Wait(timeout.ToDeadLine()); - } + bool Wait(TDuration timeout) const { + return Wait(timeout.ToDeadLine()); + } - bool Wait(TInstant deadline) const { + bool Wait(TInstant deadline) const { TSystemEvent* readyEvent = nullptr; - - with_lock (StateLock) { - int state = AtomicGet(State); - if (state != NotReady) { - return true; - } - - if (!ReadyEvent) { + + with_lock (StateLock) { + int state = AtomicGet(State); + if (state != NotReady) { + return true; + } + + if (!ReadyEvent) { ReadyEvent.Reset(new TSystemEvent()); - } - readyEvent = ReadyEvent.Get(); - } - - Y_ASSERT(readyEvent); - return readyEvent->WaitD(deadline); + } + readyEvent = ReadyEvent.Get(); + } + + Y_ASSERT(readyEvent); + return readyEvent->WaitD(deadline); } void TryRethrowWithState(int state) const { @@ -436,53 +436,53 @@ namespace NThreading { std::rethrow_exception(Exception); } } - }; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - template <typename T> - inline void SetValueImpl(TPromise<T>& promise, const T& value) { - promise.SetValue(value); - } + template <typename T> + inline void SetValueImpl(TPromise<T>& promise, const T& value) { + promise.SetValue(value); + } - template <typename T> - inline void SetValueImpl(TPromise<T>& promise, T&& value) { - promise.SetValue(std::move(value)); + template <typename T> + inline void SetValueImpl(TPromise<T>& promise, T&& value) { + promise.SetValue(std::move(value)); } - template <typename T> + template <typename T> inline void SetValueImpl(TPromise<T>& promise, const TFuture<T>& future, std::enable_if_t<!std::is_void<T>::value, bool> = false) { - future.Subscribe([=](const TFuture<T>& f) mutable { + future.Subscribe([=](const TFuture<T>& f) mutable { T const* value; - try { + try { value = &f.GetValue(); - } catch (...) { - promise.SetException(std::current_exception()); + } catch (...) { + promise.SetException(std::current_exception()); return; - } + } promise.SetValue(*value); - }); + }); } template <typename T> inline void SetValueImpl(TPromise<void>& promise, const TFuture<T>& future) { future.Subscribe([=](const TFuture<T>& f) mutable { - try { + try { f.TryRethrow(); - } catch (...) { - promise.SetException(std::current_exception()); + } catch (...) { + promise.SetException(std::current_exception()); return; - } + } promise.SetValue(); - }); - } - - template <typename T, typename F> - inline void SetValue(TPromise<T>& promise, F&& func) { - try { - SetValueImpl(promise, func()); - } catch (...) { + }); + } + + template <typename T, typename F> + inline void SetValue(TPromise<T>& promise, F&& func) { + try { + SetValueImpl(promise, func()); + } catch (...) { const bool success = promise.TrySetException(std::current_exception()); if (Y_UNLIKELY(!success)) { throw; @@ -490,21 +490,21 @@ namespace NThreading { } } - template <typename F> - inline void SetValue(TPromise<void>& promise, F&& func, - std::enable_if_t<std::is_void<TFunctionResult<F>>::value, bool> = false) { - try { - func(); - } catch (...) { - promise.SetException(std::current_exception()); + template <typename F> + inline void SetValue(TPromise<void>& promise, F&& func, + std::enable_if_t<std::is_void<TFunctionResult<F>>::value, bool> = false) { + try { + func(); + } catch (...) { + promise.SetException(std::current_exception()); return; } promise.SetValue(); } - } + } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// class TFutureStateId { private: @@ -532,45 +532,45 @@ namespace NThreading { //////////////////////////////////////////////////////////////////////////////// - template <typename T> + template <typename T> inline TFuture<T>::TFuture(const TIntrusivePtr<TFutureState>& state) noexcept - : State(state) + : State(state) { } - template <typename T> - inline void TFuture<T>::Swap(TFuture<T>& other) { - State.Swap(other.State); - } + template <typename T> + inline void TFuture<T>::Swap(TFuture<T>& other) { + State.Swap(other.State); + } - template <typename T> - inline bool TFuture<T>::HasValue() const { - return State && State->HasValue(); - } + template <typename T> + inline bool TFuture<T>::HasValue() const { + return State && State->HasValue(); + } - template <typename T> - inline const T& TFuture<T>::GetValue(TDuration timeout) const { - EnsureInitialized(); - return State->GetValue(timeout); + template <typename T> + inline const T& TFuture<T>::GetValue(TDuration timeout) const { + EnsureInitialized(); + return State->GetValue(timeout); } - template <typename T> - inline T TFuture<T>::ExtractValue(TDuration timeout) { - EnsureInitialized(); - return State->ExtractValue(timeout); - } + template <typename T> + inline T TFuture<T>::ExtractValue(TDuration timeout) { + EnsureInitialized(); + return State->ExtractValue(timeout); + } - template <typename T> - inline const T& TFuture<T>::GetValueSync() const { - return GetValue(TDuration::Max()); - } + template <typename T> + inline const T& TFuture<T>::GetValueSync() const { + return GetValue(TDuration::Max()); + } - template <typename T> - inline T TFuture<T>::ExtractValueSync() { - return ExtractValue(TDuration::Max()); - } + template <typename T> + inline T TFuture<T>::ExtractValueSync() { + return ExtractValue(TDuration::Max()); + } - template <typename T> + template <typename T> inline void TFuture<T>::TryRethrow() const { if (State) { State->TryRethrow(); @@ -578,40 +578,40 @@ namespace NThreading { } template <typename T> - inline bool TFuture<T>::HasException() const { - return State && State->HasException(); - } - - template <typename T> - inline void TFuture<T>::Wait() const { - EnsureInitialized(); - return State->Wait(); - } - - template <typename T> - inline bool TFuture<T>::Wait(TDuration timeout) const { - EnsureInitialized(); - return State->Wait(timeout); - } - - template <typename T> - inline bool TFuture<T>::Wait(TInstant deadline) const { - EnsureInitialized(); - return State->Wait(deadline); - } - - template <typename T> - template <typename F> - inline const TFuture<T>& TFuture<T>::Subscribe(F&& func) const { - EnsureInitialized(); - if (!State->Subscribe(std::forward<F>(func))) { - func(*this); - } - return *this; - } - - template <typename T> - template <typename F> + inline bool TFuture<T>::HasException() const { + return State && State->HasException(); + } + + template <typename T> + inline void TFuture<T>::Wait() const { + EnsureInitialized(); + return State->Wait(); + } + + template <typename T> + inline bool TFuture<T>::Wait(TDuration timeout) const { + EnsureInitialized(); + return State->Wait(timeout); + } + + template <typename T> + inline bool TFuture<T>::Wait(TInstant deadline) const { + EnsureInitialized(); + return State->Wait(deadline); + } + + template <typename T> + template <typename F> + inline const TFuture<T>& TFuture<T>::Subscribe(F&& func) const { + EnsureInitialized(); + if (!State->Subscribe(std::forward<F>(func))) { + func(*this); + } + return *this; + } + + template <typename T> + template <typename F> inline const TFuture<T>& TFuture<T>::NoexceptSubscribe(F&& func) const noexcept { return Subscribe(std::forward<F>(func)); } @@ -623,59 +623,59 @@ namespace NThreading { auto promise = NewPromise<TFutureType<TFutureCallResult<F, T>>>(); Subscribe([promise, func = std::forward<F>(func)](const TFuture<T>& future) mutable { NImpl::SetValue(promise, [&]() { return func(future); }); - }); - return promise; - } - - template <typename T> - inline TFuture<void> TFuture<T>::IgnoreResult() const { - auto promise = NewPromise(); - Subscribe([=](const TFuture<T>& future) mutable { + }); + return promise; + } + + template <typename T> + inline TFuture<void> TFuture<T>::IgnoreResult() const { + auto promise = NewPromise(); + Subscribe([=](const TFuture<T>& future) mutable { NImpl::SetValueImpl(promise, future); - }); - return promise; - } + }); + return promise; + } - template <typename T> - inline bool TFuture<T>::Initialized() const { - return bool(State); + template <typename T> + inline bool TFuture<T>::Initialized() const { + return bool(State); } - template <typename T> + template <typename T> inline TMaybe<TFutureStateId> TFuture<T>::StateId() const noexcept { return State != nullptr ? MakeMaybe<TFutureStateId>(*State) : Nothing(); } template <typename T> - inline void TFuture<T>::EnsureInitialized() const { - if (!State) { - ythrow TFutureException() << "state not initialized"; + inline void TFuture<T>::EnsureInitialized() const { + if (!State) { + ythrow TFutureException() << "state not initialized"; } - } + } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// inline TFuture<void>::TFuture(const TIntrusivePtr<TFutureState>& state) noexcept - : State(state) - { - } + : State(state) + { + } - inline void TFuture<void>::Swap(TFuture<void>& other) { - State.Swap(other.State); - } + inline void TFuture<void>::Swap(TFuture<void>& other) { + State.Swap(other.State); + } - inline bool TFuture<void>::HasValue() const { - return State && State->HasValue(); - } + inline bool TFuture<void>::HasValue() const { + return State && State->HasValue(); + } - inline void TFuture<void>::GetValue(TDuration timeout) const { - EnsureInitialized(); - State->GetValue(timeout); - } + inline void TFuture<void>::GetValue(TDuration timeout) const { + EnsureInitialized(); + State->GetValue(timeout); + } - inline void TFuture<void>::GetValueSync() const { - GetValue(TDuration::Max()); - } + inline void TFuture<void>::GetValueSync() const { + GetValue(TDuration::Max()); + } inline void TFuture<void>::TryRethrow() const { if (State) { @@ -683,35 +683,35 @@ namespace NThreading { } } - inline bool TFuture<void>::HasException() const { - return State && State->HasException(); - } + inline bool TFuture<void>::HasException() const { + return State && State->HasException(); + } - inline void TFuture<void>::Wait() const { - EnsureInitialized(); - return State->Wait(); - } + inline void TFuture<void>::Wait() const { + EnsureInitialized(); + return State->Wait(); + } - inline bool TFuture<void>::Wait(TDuration timeout) const { - EnsureInitialized(); - return State->Wait(timeout); - } + inline bool TFuture<void>::Wait(TDuration timeout) const { + EnsureInitialized(); + return State->Wait(timeout); + } - inline bool TFuture<void>::Wait(TInstant deadline) const { - EnsureInitialized(); - return State->Wait(deadline); - } + inline bool TFuture<void>::Wait(TInstant deadline) const { + EnsureInitialized(); + return State->Wait(deadline); + } - template <typename F> - inline const TFuture<void>& TFuture<void>::Subscribe(F&& func) const { - EnsureInitialized(); - if (!State->Subscribe(std::forward<F>(func))) { - func(*this); - } - return *this; - } + template <typename F> + inline const TFuture<void>& TFuture<void>::Subscribe(F&& func) const { + EnsureInitialized(); + if (!State->Subscribe(std::forward<F>(func))) { + func(*this); + } + return *this; + } - template <typename F> + template <typename F> inline const TFuture<void>& TFuture<void>::NoexceptSubscribe(F&& func) const noexcept { return Subscribe(std::forward<F>(func)); } @@ -722,82 +722,82 @@ namespace NThreading { auto promise = NewPromise<TFutureType<TFutureCallResult<F, void>>>(); Subscribe([promise, func = std::forward<F>(func)](const TFuture<void>& future) mutable { NImpl::SetValue(promise, [&]() { return func(future); }); - }); - return promise; - } - - template <typename R> - inline TFuture<R> TFuture<void>::Return(const R& value) const { - auto promise = NewPromise<R>(); - Subscribe([=](const TFuture<void>& future) mutable { - try { + }); + return promise; + } + + template <typename R> + inline TFuture<R> TFuture<void>::Return(const R& value) const { + auto promise = NewPromise<R>(); + Subscribe([=](const TFuture<void>& future) mutable { + try { future.TryRethrow(); - } catch (...) { - promise.SetException(std::current_exception()); + } catch (...) { + promise.SetException(std::current_exception()); return; - } + } promise.SetValue(value); - }); - return promise; + }); + return promise; } - inline bool TFuture<void>::Initialized() const { - return bool(State); - } + inline bool TFuture<void>::Initialized() const { + return bool(State); + } inline TMaybe<TFutureStateId> TFuture<void>::StateId() const noexcept { return State != nullptr ? MakeMaybe<TFutureStateId>(*State) : Nothing(); } - inline void TFuture<void>::EnsureInitialized() const { - if (!State) { - ythrow TFutureException() << "state not initialized"; + inline void TFuture<void>::EnsureInitialized() const { + if (!State) { + ythrow TFutureException() << "state not initialized"; } - } + } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - template <typename T> + template <typename T> inline TPromise<T>::TPromise(const TIntrusivePtr<TFutureState>& state) noexcept - : State(state) - { - } - - template <typename T> - inline void TPromise<T>::Swap(TPromise<T>& other) { - State.Swap(other.State); - } - - template <typename T> - inline const T& TPromise<T>::GetValue() const { - EnsureInitialized(); - return State->GetValue(); - } - - template <typename T> - inline T TPromise<T>::ExtractValue() { - EnsureInitialized(); - return State->ExtractValue(); - } - - template <typename T> - inline bool TPromise<T>::HasValue() const { - return State && State->HasValue(); - } - - template <typename T> - inline void TPromise<T>::SetValue(const T& value) { - EnsureInitialized(); - State->SetValue(value); - } - - template <typename T> - inline void TPromise<T>::SetValue(T&& value) { - EnsureInitialized(); - State->SetValue(std::move(value)); - } - - template <typename T> + : State(state) + { + } + + template <typename T> + inline void TPromise<T>::Swap(TPromise<T>& other) { + State.Swap(other.State); + } + + template <typename T> + inline const T& TPromise<T>::GetValue() const { + EnsureInitialized(); + return State->GetValue(); + } + + template <typename T> + inline T TPromise<T>::ExtractValue() { + EnsureInitialized(); + return State->ExtractValue(); + } + + template <typename T> + inline bool TPromise<T>::HasValue() const { + return State && State->HasValue(); + } + + template <typename T> + inline void TPromise<T>::SetValue(const T& value) { + EnsureInitialized(); + State->SetValue(value); + } + + template <typename T> + inline void TPromise<T>::SetValue(T&& value) { + EnsureInitialized(); + State->SetValue(std::move(value)); + } + + template <typename T> inline bool TPromise<T>::TrySetValue(const T& value) { EnsureInitialized(); return State->TrySetValue(value); @@ -817,75 +817,75 @@ namespace NThreading { } template <typename T> - inline bool TPromise<T>::HasException() const { - return State && State->HasException(); - } + inline bool TPromise<T>::HasException() const { + return State && State->HasException(); + } - template <typename T> - inline void TPromise<T>::SetException(const TString& e) { - EnsureInitialized(); - State->SetException(std::make_exception_ptr(yexception() << e)); - } + template <typename T> + inline void TPromise<T>::SetException(const TString& e) { + EnsureInitialized(); + State->SetException(std::make_exception_ptr(yexception() << e)); + } - template <typename T> - inline void TPromise<T>::SetException(std::exception_ptr e) { - EnsureInitialized(); - State->SetException(std::move(e)); - } + template <typename T> + inline void TPromise<T>::SetException(std::exception_ptr e) { + EnsureInitialized(); + State->SetException(std::move(e)); + } - template <typename T> + template <typename T> inline bool TPromise<T>::TrySetException(std::exception_ptr e) { EnsureInitialized(); return State->TrySetException(std::move(e)); } template <typename T> - inline TFuture<T> TPromise<T>::GetFuture() const { - EnsureInitialized(); - return TFuture<T>(State); - } + inline TFuture<T> TPromise<T>::GetFuture() const { + EnsureInitialized(); + return TFuture<T>(State); + } - template <typename T> - inline TPromise<T>::operator TFuture<T>() const { - return GetFuture(); - } + template <typename T> + inline TPromise<T>::operator TFuture<T>() const { + return GetFuture(); + } - template <typename T> - inline bool TPromise<T>::Initialized() const { - return bool(State); - } + template <typename T> + inline bool TPromise<T>::Initialized() const { + return bool(State); + } - template <typename T> - inline void TPromise<T>::EnsureInitialized() const { - if (!State) { - ythrow TFutureException() << "state not initialized"; - } - } + template <typename T> + inline void TPromise<T>::EnsureInitialized() const { + if (!State) { + ythrow TFutureException() << "state not initialized"; + } + } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// inline TPromise<void>::TPromise(const TIntrusivePtr<TFutureState>& state) noexcept - : State(state) - { - } + : State(state) + { + } - inline void TPromise<void>::Swap(TPromise<void>& other) { - State.Swap(other.State); - } + inline void TPromise<void>::Swap(TPromise<void>& other) { + State.Swap(other.State); + } - inline void TPromise<void>::GetValue() const { - EnsureInitialized(); - State->GetValue(); - } + inline void TPromise<void>::GetValue() const { + EnsureInitialized(); + State->GetValue(); + } - inline bool TPromise<void>::HasValue() const { - return State && State->HasValue(); - } + inline bool TPromise<void>::HasValue() const { + return State && State->HasValue(); + } - inline void TPromise<void>::SetValue() { - EnsureInitialized(); - State->SetValue(); - } + inline void TPromise<void>::SetValue() { + EnsureInitialized(); + State->SetValue(); + } inline bool TPromise<void>::TrySetValue() { EnsureInitialized(); @@ -898,78 +898,78 @@ namespace NThreading { } } - inline bool TPromise<void>::HasException() const { - return State && State->HasException(); - } + inline bool TPromise<void>::HasException() const { + return State && State->HasException(); + } - inline void TPromise<void>::SetException(const TString& e) { - EnsureInitialized(); - State->SetException(std::make_exception_ptr(yexception() << e)); - } + inline void TPromise<void>::SetException(const TString& e) { + EnsureInitialized(); + State->SetException(std::make_exception_ptr(yexception() << e)); + } - inline void TPromise<void>::SetException(std::exception_ptr e) { - EnsureInitialized(); - State->SetException(std::move(e)); - } + inline void TPromise<void>::SetException(std::exception_ptr e) { + EnsureInitialized(); + State->SetException(std::move(e)); + } inline bool TPromise<void>::TrySetException(std::exception_ptr e) { EnsureInitialized(); return State->TrySetException(std::move(e)); } - inline TFuture<void> TPromise<void>::GetFuture() const { - EnsureInitialized(); - return TFuture<void>(State); - } + inline TFuture<void> TPromise<void>::GetFuture() const { + EnsureInitialized(); + return TFuture<void>(State); + } - inline TPromise<void>::operator TFuture<void>() const { - return GetFuture(); - } + inline TPromise<void>::operator TFuture<void>() const { + return GetFuture(); + } - inline bool TPromise<void>::Initialized() const { - return bool(State); - } + inline bool TPromise<void>::Initialized() const { + return bool(State); + } - inline void TPromise<void>::EnsureInitialized() const { - if (!State) { - ythrow TFutureException() << "state not initialized"; - } - } + inline void TPromise<void>::EnsureInitialized() const { + if (!State) { + ythrow TFutureException() << "state not initialized"; + } + } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - template <typename T> - inline TPromise<T> NewPromise() { - return {new NImpl::TFutureState<T>()}; + template <typename T> + inline TPromise<T> NewPromise() { + return {new NImpl::TFutureState<T>()}; } - inline TPromise<void> NewPromise() { - return {new NImpl::TFutureState<void>()}; - } + inline TPromise<void> NewPromise() { + return {new NImpl::TFutureState<void>()}; + } - template <typename T> - inline TFuture<T> MakeFuture(const T& value) { - return {new NImpl::TFutureState<T>(value)}; - } + template <typename T> + inline TFuture<T> MakeFuture(const T& value) { + return {new NImpl::TFutureState<T>(value)}; + } - template <typename T> - inline TFuture<std::remove_reference_t<T>> MakeFuture(T&& value) { - return {new NImpl::TFutureState<std::remove_reference_t<T>>(std::forward<T>(value))}; - } + template <typename T> + inline TFuture<std::remove_reference_t<T>> MakeFuture(T&& value) { + return {new NImpl::TFutureState<std::remove_reference_t<T>>(std::forward<T>(value))}; + } - template <typename T> - inline TFuture<T> MakeFuture() { - struct TCache { - TFuture<T> Instance{new NImpl::TFutureState<T>(Default<T>())}; + template <typename T> + inline TFuture<T> MakeFuture() { + struct TCache { + TFuture<T> Instance{new NImpl::TFutureState<T>(Default<T>())}; TCache() { // Immediately advance state from ValueSet to ValueRead. // This should prevent corrupting shared value with an ExtractValue() call. Y_UNUSED(Instance.GetValue()); } - }; - return Singleton<TCache>()->Instance; - } + }; + return Singleton<TCache>()->Instance; + } template <typename T> inline TFuture<T> MakeErrorFuture(std::exception_ptr exception) @@ -977,10 +977,10 @@ namespace NThreading { return {new NImpl::TFutureState<T>(std::move(exception), NImpl::TError::Error)}; } - inline TFuture<void> MakeFuture() { - struct TCache { - TFuture<void> Instance{new NImpl::TFutureState<void>(true)}; - }; - return Singleton<TCache>()->Instance; - } + inline TFuture<void> MakeFuture() { + struct TCache { + TFuture<void> Instance{new NImpl::TFutureState<void>(true)}; + }; + return Singleton<TCache>()->Instance; + } } diff --git a/library/cpp/threading/future/core/future.h b/library/cpp/threading/future/core/future.h index 2e82bb953e..12623389ca 100644 --- a/library/cpp/threading/future/core/future.h +++ b/library/cpp/threading/future/core/future.h @@ -12,51 +12,51 @@ #include <util/system/spinlock.h> namespace NThreading { - //////////////////////////////////////////////////////////////////////////////// - - struct TFutureException: public yexception {}; - - // creates unset promise - template <typename T> - TPromise<T> NewPromise(); - TPromise<void> NewPromise(); - - // creates preset future - template <typename T> - TFuture<T> MakeFuture(const T& value); - template <typename T> - TFuture<std::remove_reference_t<T>> MakeFuture(T&& value); - template <typename T> - TFuture<T> MakeFuture(); + //////////////////////////////////////////////////////////////////////////////// + + struct TFutureException: public yexception {}; + + // creates unset promise + template <typename T> + TPromise<T> NewPromise(); + TPromise<void> NewPromise(); + + // creates preset future + template <typename T> + TFuture<T> MakeFuture(const T& value); + template <typename T> + TFuture<std::remove_reference_t<T>> MakeFuture(T&& value); + template <typename T> + TFuture<T> MakeFuture(); template <typename T> TFuture<T> MakeErrorFuture(std::exception_ptr exception); - TFuture<void> MakeFuture(); + TFuture<void> MakeFuture(); - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - namespace NImpl { - template <typename T> - class TFutureState; + namespace NImpl { + template <typename T> + class TFutureState; - template <typename T> - struct TFutureType { - using TType = T; - }; + template <typename T> + struct TFutureType { + using TType = T; + }; - template <typename T> - struct TFutureType<TFuture<T>> { - using TType = typename TFutureType<T>::TType; - }; + template <typename T> + struct TFutureType<TFuture<T>> { + using TType = typename TFutureType<T>::TType; + }; template <typename F, typename T> struct TFutureCallResult { // NOTE: separate class for msvc compatibility using TType = decltype(std::declval<F&>()(std::declval<const TFuture<T>&>())); }; - } + } - template <typename F> - using TFutureType = typename NImpl::TFutureType<F>::TType; + template <typename F> + using TFutureType = typename NImpl::TFutureType<F>::TType; template <typename F, typename T> using TFutureCallResult = typename NImpl::TFutureCallResult<F, T>::TType; @@ -64,16 +64,16 @@ namespace NThreading { //! Type of the future/promise state identifier class TFutureStateId; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - template <typename T> - class TFuture { - using TFutureState = NImpl::TFutureState<T>; + template <typename T> + class TFuture { + using TFutureState = NImpl::TFutureState<T>; - private: - TIntrusivePtr<TFutureState> State; + private: + TIntrusivePtr<TFutureState> State; - public: + public: using value_type = T; TFuture() noexcept = default; @@ -83,54 +83,54 @@ namespace NThreading { TFuture<T>& operator=(const TFuture<T>& other) noexcept = default; TFuture<T>& operator=(TFuture<T>&& other) noexcept = default; - void Swap(TFuture<T>& other); + void Swap(TFuture<T>& other); - bool Initialized() const; + bool Initialized() const; - bool HasValue() const; - const T& GetValue(TDuration timeout = TDuration::Zero()) const; - const T& GetValueSync() const; - T ExtractValue(TDuration timeout = TDuration::Zero()); - T ExtractValueSync(); + bool HasValue() const; + const T& GetValue(TDuration timeout = TDuration::Zero()) const; + const T& GetValueSync() const; + T ExtractValue(TDuration timeout = TDuration::Zero()); + T ExtractValueSync(); void TryRethrow() const; - bool HasException() const; + bool HasException() const; - void Wait() const; - bool Wait(TDuration timeout) const; - bool Wait(TInstant deadline) const; + void Wait() const; + bool Wait(TDuration timeout) const; + bool Wait(TInstant deadline) const; - template <typename F> - const TFuture<T>& Subscribe(F&& callback) const; + template <typename F> + const TFuture<T>& Subscribe(F&& callback) const; // precondition: EnsureInitialized() passes // postcondition: std::terminate is highly unlikely - template <typename F> + template <typename F> const TFuture<T>& NoexceptSubscribe(F&& callback) const noexcept; template <typename F> TFuture<TFutureType<TFutureCallResult<F, T>>> Apply(F&& func) const; - TFuture<void> IgnoreResult() const; + TFuture<void> IgnoreResult() const; //! If the future is initialized returns the future state identifier. Otherwise returns an empty optional /** The state identifier is guaranteed to be unique during the future state lifetime and could be reused after its death **/ TMaybe<TFutureStateId> StateId() const noexcept; - void EnsureInitialized() const; - }; + void EnsureInitialized() const; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - template <> - class TFuture<void> { - using TFutureState = NImpl::TFutureState<void>; + template <> + class TFuture<void> { + using TFutureState = NImpl::TFutureState<void>; - private: + private: TIntrusivePtr<TFutureState> State = nullptr; - public: + public: using value_type = void; TFuture() noexcept = default; @@ -140,34 +140,34 @@ namespace NThreading { TFuture<void>& operator=(const TFuture<void>& other) noexcept = default; TFuture<void>& operator=(TFuture<void>&& other) noexcept = default; - void Swap(TFuture<void>& other); + void Swap(TFuture<void>& other); - bool Initialized() const; + bool Initialized() const; - bool HasValue() const; - void GetValue(TDuration timeout = TDuration::Zero()) const; - void GetValueSync() const; + bool HasValue() const; + void GetValue(TDuration timeout = TDuration::Zero()) const; + void GetValueSync() const; void TryRethrow() const; - bool HasException() const; + bool HasException() const; - void Wait() const; - bool Wait(TDuration timeout) const; - bool Wait(TInstant deadline) const; + void Wait() const; + bool Wait(TDuration timeout) const; + bool Wait(TInstant deadline) const; - template <typename F> - const TFuture<void>& Subscribe(F&& callback) const; + template <typename F> + const TFuture<void>& Subscribe(F&& callback) const; // precondition: EnsureInitialized() passes // postcondition: std::terminate is highly unlikely - template <typename F> + template <typename F> const TFuture<void>& NoexceptSubscribe(F&& callback) const noexcept; template <typename F> TFuture<TFutureType<TFutureCallResult<F, void>>> Apply(F&& func) const; - template <typename R> - TFuture<R> Return(const R& value) const; + template <typename R> + TFuture<R> Return(const R& value) const; TFuture<void> IgnoreResult() const { return *this; @@ -178,19 +178,19 @@ namespace NThreading { **/ TMaybe<TFutureStateId> StateId() const noexcept; - void EnsureInitialized() const; - }; + void EnsureInitialized() const; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - template <typename T> - class TPromise { - using TFutureState = NImpl::TFutureState<T>; + template <typename T> + class TPromise { + using TFutureState = NImpl::TFutureState<T>; - private: + private: TIntrusivePtr<TFutureState> State = nullptr; - public: + public: TPromise() noexcept = default; TPromise(const TPromise<T>& other) noexcept = default; TPromise(TPromise<T>&& other) noexcept = default; @@ -198,43 +198,43 @@ namespace NThreading { TPromise<T>& operator=(const TPromise<T>& other) noexcept = default; TPromise<T>& operator=(TPromise<T>&& other) noexcept = default; - void Swap(TPromise<T>& other); + void Swap(TPromise<T>& other); - bool Initialized() const; + bool Initialized() const; - bool HasValue() const; - const T& GetValue() const; - T ExtractValue(); + bool HasValue() const; + const T& GetValue() const; + T ExtractValue(); - void SetValue(const T& value); - void SetValue(T&& value); + void SetValue(const T& value); + void SetValue(T&& value); bool TrySetValue(const T& value); bool TrySetValue(T&& value); void TryRethrow() const; - bool HasException() const; - void SetException(const TString& e); - void SetException(std::exception_ptr e); + bool HasException() const; + void SetException(const TString& e); + void SetException(std::exception_ptr e); bool TrySetException(std::exception_ptr e); - TFuture<T> GetFuture() const; - operator TFuture<T>() const; + TFuture<T> GetFuture() const; + operator TFuture<T>() const; - private: - void EnsureInitialized() const; - }; + private: + void EnsureInitialized() const; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - template <> - class TPromise<void> { - using TFutureState = NImpl::TFutureState<void>; + template <> + class TPromise<void> { + using TFutureState = NImpl::TFutureState<void>; - private: - TIntrusivePtr<TFutureState> State; + private: + TIntrusivePtr<TFutureState> State; - public: + public: TPromise() noexcept = default; TPromise(const TPromise<void>& other) noexcept = default; TPromise(TPromise<void>&& other) noexcept = default; @@ -242,30 +242,30 @@ namespace NThreading { TPromise<void>& operator=(const TPromise<void>& other) noexcept = default; TPromise<void>& operator=(TPromise<void>&& other) noexcept = default; - void Swap(TPromise<void>& other); + void Swap(TPromise<void>& other); - bool Initialized() const; + bool Initialized() const; - bool HasValue() const; - void GetValue() const; + bool HasValue() const; + void GetValue() const; - void SetValue(); + void SetValue(); bool TrySetValue(); void TryRethrow() const; - bool HasException() const; - void SetException(const TString& e); - void SetException(std::exception_ptr e); + bool HasException() const; + void SetException(const TString& e); + void SetException(std::exception_ptr e); bool TrySetException(std::exception_ptr e); - TFuture<void> GetFuture() const; - operator TFuture<void>() const; + TFuture<void> GetFuture() const; + operator TFuture<void>() const; - private: - void EnsureInitialized() const; - }; + private: + void EnsureInitialized() const; + }; -} +} #define INCLUDE_FUTURE_INL_H #include "future-inl.h" diff --git a/library/cpp/threading/future/future_ut.cpp b/library/cpp/threading/future/future_ut.cpp index 05950a568d..636b113f2f 100644 --- a/library/cpp/threading/future/future_ut.cpp +++ b/library/cpp/threading/future/future_ut.cpp @@ -62,180 +62,180 @@ namespace { } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// Y_UNIT_TEST_SUITE(TFutureTest) { Y_UNIT_TEST(ShouldInitiallyHasNoValue) { - TPromise<int> promise; - UNIT_ASSERT(!promise.HasValue()); + TPromise<int> promise; + UNIT_ASSERT(!promise.HasValue()); - promise = NewPromise<int>(); - UNIT_ASSERT(!promise.HasValue()); + promise = NewPromise<int>(); + UNIT_ASSERT(!promise.HasValue()); - TFuture<int> future; - UNIT_ASSERT(!future.HasValue()); + TFuture<int> future; + UNIT_ASSERT(!future.HasValue()); - future = promise.GetFuture(); - UNIT_ASSERT(!future.HasValue()); - } + future = promise.GetFuture(); + UNIT_ASSERT(!future.HasValue()); + } Y_UNIT_TEST(ShouldInitiallyHasNoValueVoid) { - TPromise<void> promise; - UNIT_ASSERT(!promise.HasValue()); + TPromise<void> promise; + UNIT_ASSERT(!promise.HasValue()); - promise = NewPromise(); - UNIT_ASSERT(!promise.HasValue()); + promise = NewPromise(); + UNIT_ASSERT(!promise.HasValue()); - TFuture<void> future; - UNIT_ASSERT(!future.HasValue()); + TFuture<void> future; + UNIT_ASSERT(!future.HasValue()); - future = promise.GetFuture(); - UNIT_ASSERT(!future.HasValue()); - } + future = promise.GetFuture(); + UNIT_ASSERT(!future.HasValue()); + } Y_UNIT_TEST(ShouldStoreValue) { - TPromise<int> promise = NewPromise<int>(); - promise.SetValue(123); - UNIT_ASSERT(promise.HasValue()); - UNIT_ASSERT_EQUAL(promise.GetValue(), 123); + TPromise<int> promise = NewPromise<int>(); + promise.SetValue(123); + UNIT_ASSERT(promise.HasValue()); + UNIT_ASSERT_EQUAL(promise.GetValue(), 123); - TFuture<int> future = promise.GetFuture(); - UNIT_ASSERT(future.HasValue()); - UNIT_ASSERT_EQUAL(future.GetValue(), 123); + TFuture<int> future = promise.GetFuture(); + UNIT_ASSERT(future.HasValue()); + UNIT_ASSERT_EQUAL(future.GetValue(), 123); - future = MakeFuture(345); - UNIT_ASSERT(future.HasValue()); - UNIT_ASSERT_EQUAL(future.GetValue(), 345); - } + future = MakeFuture(345); + UNIT_ASSERT(future.HasValue()); + UNIT_ASSERT_EQUAL(future.GetValue(), 345); + } Y_UNIT_TEST(ShouldStoreValueVoid) { - TPromise<void> promise = NewPromise(); - promise.SetValue(); - UNIT_ASSERT(promise.HasValue()); + TPromise<void> promise = NewPromise(); + promise.SetValue(); + UNIT_ASSERT(promise.HasValue()); - TFuture<void> future = promise.GetFuture(); - UNIT_ASSERT(future.HasValue()); + TFuture<void> future = promise.GetFuture(); + UNIT_ASSERT(future.HasValue()); - future = MakeFuture(); - UNIT_ASSERT(future.HasValue()); - } + future = MakeFuture(); + UNIT_ASSERT(future.HasValue()); + } - struct TTestCallback { - int Value; + struct TTestCallback { + int Value; - TTestCallback(int value) - : Value(value) - { - } + TTestCallback(int value) + : Value(value) + { + } - void Callback(const TFuture<int>& future) { - Value += future.GetValue(); - } + void Callback(const TFuture<int>& future) { + Value += future.GetValue(); + } - int Func(const TFuture<int>& future) { - return (Value += future.GetValue()); - } + int Func(const TFuture<int>& future) { + return (Value += future.GetValue()); + } - void VoidFunc(const TFuture<int>& future) { - future.GetValue(); - } + void VoidFunc(const TFuture<int>& future) { + future.GetValue(); + } - TFuture<int> FutureFunc(const TFuture<int>& future) { - return MakeFuture(Value += future.GetValue()); - } + TFuture<int> FutureFunc(const TFuture<int>& future) { + return MakeFuture(Value += future.GetValue()); + } - TPromise<void> Signal = NewPromise(); - TFuture<void> FutureVoidFunc(const TFuture<int>& future) { - future.GetValue(); - return Signal; - } - }; + TPromise<void> Signal = NewPromise(); + TFuture<void> FutureVoidFunc(const TFuture<int>& future) { + future.GetValue(); + return Signal; + } + }; Y_UNIT_TEST(ShouldInvokeCallback) { - TPromise<int> promise = NewPromise<int>(); + TPromise<int> promise = NewPromise<int>(); - TTestCallback callback(123); - TFuture<int> future = promise.GetFuture() - .Subscribe([&](const TFuture<int>& theFuture) { return callback.Callback(theFuture); }); + TTestCallback callback(123); + TFuture<int> future = promise.GetFuture() + .Subscribe([&](const TFuture<int>& theFuture) { return callback.Callback(theFuture); }); - promise.SetValue(456); - UNIT_ASSERT_EQUAL(future.GetValue(), 456); - UNIT_ASSERT_EQUAL(callback.Value, 123 + 456); - } + promise.SetValue(456); + UNIT_ASSERT_EQUAL(future.GetValue(), 456); + UNIT_ASSERT_EQUAL(callback.Value, 123 + 456); + } Y_UNIT_TEST(ShouldApplyFunc) { - TPromise<int> promise = NewPromise<int>(); + TPromise<int> promise = NewPromise<int>(); - TTestCallback callback(123); - TFuture<int> future = promise.GetFuture() + TTestCallback callback(123); + TFuture<int> future = promise.GetFuture() .Apply([&](const auto& theFuture) { return callback.Func(theFuture); }); - promise.SetValue(456); - UNIT_ASSERT_EQUAL(future.GetValue(), 123 + 456); - UNIT_ASSERT_EQUAL(callback.Value, 123 + 456); - } + promise.SetValue(456); + UNIT_ASSERT_EQUAL(future.GetValue(), 123 + 456); + UNIT_ASSERT_EQUAL(callback.Value, 123 + 456); + } Y_UNIT_TEST(ShouldApplyVoidFunc) { - TPromise<int> promise = NewPromise<int>(); + TPromise<int> promise = NewPromise<int>(); - TTestCallback callback(123); - TFuture<void> future = promise.GetFuture() + TTestCallback callback(123); + TFuture<void> future = promise.GetFuture() .Apply([&](const auto& theFuture) { return callback.VoidFunc(theFuture); }); - promise.SetValue(456); - UNIT_ASSERT(future.HasValue()); - } + promise.SetValue(456); + UNIT_ASSERT(future.HasValue()); + } Y_UNIT_TEST(ShouldApplyFutureFunc) { - TPromise<int> promise = NewPromise<int>(); + TPromise<int> promise = NewPromise<int>(); - TTestCallback callback(123); - TFuture<int> future = promise.GetFuture() + TTestCallback callback(123); + TFuture<int> future = promise.GetFuture() .Apply([&](const auto& theFuture) { return callback.FutureFunc(theFuture); }); - promise.SetValue(456); - UNIT_ASSERT_EQUAL(future.GetValue(), 123 + 456); - UNIT_ASSERT_EQUAL(callback.Value, 123 + 456); - } + promise.SetValue(456); + UNIT_ASSERT_EQUAL(future.GetValue(), 123 + 456); + UNIT_ASSERT_EQUAL(callback.Value, 123 + 456); + } Y_UNIT_TEST(ShouldApplyFutureVoidFunc) { - TPromise<int> promise = NewPromise<int>(); + TPromise<int> promise = NewPromise<int>(); - TTestCallback callback(123); - TFuture<void> future = promise.GetFuture() + TTestCallback callback(123); + TFuture<void> future = promise.GetFuture() .Apply([&](const auto& theFuture) { return callback.FutureVoidFunc(theFuture); }); - promise.SetValue(456); - UNIT_ASSERT(!future.HasValue()); + promise.SetValue(456); + UNIT_ASSERT(!future.HasValue()); - callback.Signal.SetValue(); - UNIT_ASSERT(future.HasValue()); - } + callback.Signal.SetValue(); + UNIT_ASSERT(future.HasValue()); + } Y_UNIT_TEST(ShouldIgnoreResultIfAsked) { - TPromise<int> promise = NewPromise<int>(); + TPromise<int> promise = NewPromise<int>(); - TTestCallback callback(123); - TFuture<int> future = promise.GetFuture().IgnoreResult().Return(42); + TTestCallback callback(123); + TFuture<int> future = promise.GetFuture().IgnoreResult().Return(42); - promise.SetValue(456); - UNIT_ASSERT_EQUAL(future.GetValue(), 42); - } + promise.SetValue(456); + UNIT_ASSERT_EQUAL(future.GetValue(), 42); + } - class TCustomException: public yexception { - }; + class TCustomException: public yexception { + }; Y_UNIT_TEST(ShouldRethrowException) { - TPromise<int> promise = NewPromise<int>(); - try { - ythrow TCustomException(); - } catch (...) { - promise.SetException(std::current_exception()); - } - - UNIT_ASSERT(!promise.HasValue()); - UNIT_ASSERT(promise.HasException()); - UNIT_ASSERT_EXCEPTION(promise.GetValue(), TCustomException); + TPromise<int> promise = NewPromise<int>(); + try { + ythrow TCustomException(); + } catch (...) { + promise.SetException(std::current_exception()); + } + + UNIT_ASSERT(!promise.HasValue()); + UNIT_ASSERT(promise.HasException()); + UNIT_ASSERT_EXCEPTION(promise.GetValue(), TCustomException); UNIT_ASSERT_EXCEPTION(promise.TryRethrow(), TCustomException); } @@ -261,36 +261,36 @@ namespace { Y_UNIT_TEST(ShouldWaitExceptionOrAll) { - TPromise<void> promise1 = NewPromise(); - TPromise<void> promise2 = NewPromise(); + TPromise<void> promise1 = NewPromise(); + TPromise<void> promise2 = NewPromise(); TFuture<void> future = WaitExceptionOrAll(promise1, promise2); - UNIT_ASSERT(!future.HasValue()); + UNIT_ASSERT(!future.HasValue()); - promise1.SetValue(); - UNIT_ASSERT(!future.HasValue()); + promise1.SetValue(); + UNIT_ASSERT(!future.HasValue()); - promise2.SetValue(); - UNIT_ASSERT(future.HasValue()); - } + promise2.SetValue(); + UNIT_ASSERT(future.HasValue()); + } Y_UNIT_TEST(ShouldWaitExceptionOrAllVector) { - TPromise<void> promise1 = NewPromise(); - TPromise<void> promise2 = NewPromise(); + TPromise<void> promise1 = NewPromise(); + TPromise<void> promise2 = NewPromise(); - TVector<TFuture<void>> promises; - promises.push_back(promise1); - promises.push_back(promise2); + TVector<TFuture<void>> promises; + promises.push_back(promise1); + promises.push_back(promise2); TFuture<void> future = WaitExceptionOrAll(promises); - UNIT_ASSERT(!future.HasValue()); + UNIT_ASSERT(!future.HasValue()); - promise1.SetValue(); - UNIT_ASSERT(!future.HasValue()); + promise1.SetValue(); + UNIT_ASSERT(!future.HasValue()); - promise2.SetValue(); - UNIT_ASSERT(future.HasValue()); - } + promise2.SetValue(); + UNIT_ASSERT(future.HasValue()); + } Y_UNIT_TEST(ShouldWaitExceptionOrAllVectorWithValueType) { TPromise<int> promise1 = NewPromise<int>(); @@ -311,47 +311,47 @@ namespace { } Y_UNIT_TEST(ShouldWaitExceptionOrAllList) { - TPromise<void> promise1 = NewPromise(); - TPromise<void> promise2 = NewPromise(); + TPromise<void> promise1 = NewPromise(); + TPromise<void> promise2 = NewPromise(); - std::list<TFuture<void>> promises; - promises.push_back(promise1); - promises.push_back(promise2); + std::list<TFuture<void>> promises; + promises.push_back(promise1); + promises.push_back(promise2); TFuture<void> future = WaitExceptionOrAll(promises); - UNIT_ASSERT(!future.HasValue()); + UNIT_ASSERT(!future.HasValue()); - promise1.SetValue(); - UNIT_ASSERT(!future.HasValue()); + promise1.SetValue(); + UNIT_ASSERT(!future.HasValue()); - promise2.SetValue(); - UNIT_ASSERT(future.HasValue()); - } + promise2.SetValue(); + UNIT_ASSERT(future.HasValue()); + } Y_UNIT_TEST(ShouldWaitExceptionOrAllVectorEmpty) { - TVector<TFuture<void>> promises; + TVector<TFuture<void>> promises; TFuture<void> future = WaitExceptionOrAll(promises); - UNIT_ASSERT(future.HasValue()); - } + UNIT_ASSERT(future.HasValue()); + } Y_UNIT_TEST(ShouldWaitAnyVector) { - TPromise<void> promise1 = NewPromise(); - TPromise<void> promise2 = NewPromise(); + TPromise<void> promise1 = NewPromise(); + TPromise<void> promise2 = NewPromise(); - TVector<TFuture<void>> promises; - promises.push_back(promise1); - promises.push_back(promise2); + TVector<TFuture<void>> promises; + promises.push_back(promise1); + promises.push_back(promise2); - TFuture<void> future = WaitAny(promises); - UNIT_ASSERT(!future.HasValue()); + TFuture<void> future = WaitAny(promises); + UNIT_ASSERT(!future.HasValue()); - promise1.SetValue(); - UNIT_ASSERT(future.HasValue()); + promise1.SetValue(); + UNIT_ASSERT(future.HasValue()); - promise2.SetValue(); - UNIT_ASSERT(future.HasValue()); - } + promise2.SetValue(); + UNIT_ASSERT(future.HasValue()); + } Y_UNIT_TEST(ShouldWaitAnyVectorWithValueType) { @@ -373,112 +373,112 @@ namespace { } Y_UNIT_TEST(ShouldWaitAnyList) { - TPromise<void> promise1 = NewPromise(); - TPromise<void> promise2 = NewPromise(); + TPromise<void> promise1 = NewPromise(); + TPromise<void> promise2 = NewPromise(); - std::list<TFuture<void>> promises; - promises.push_back(promise1); - promises.push_back(promise2); + std::list<TFuture<void>> promises; + promises.push_back(promise1); + promises.push_back(promise2); - TFuture<void> future = WaitAny(promises); - UNIT_ASSERT(!future.HasValue()); + TFuture<void> future = WaitAny(promises); + UNIT_ASSERT(!future.HasValue()); - promise1.SetValue(); - UNIT_ASSERT(future.HasValue()); + promise1.SetValue(); + UNIT_ASSERT(future.HasValue()); - promise2.SetValue(); - UNIT_ASSERT(future.HasValue()); - } + promise2.SetValue(); + UNIT_ASSERT(future.HasValue()); + } Y_UNIT_TEST(ShouldWaitAnyVectorEmpty) { - TVector<TFuture<void>> promises; + TVector<TFuture<void>> promises; - TFuture<void> future = WaitAny(promises); - UNIT_ASSERT(future.HasValue()); - } + TFuture<void> future = WaitAny(promises); + UNIT_ASSERT(future.HasValue()); + } Y_UNIT_TEST(ShouldWaitAny) { - TPromise<void> promise1 = NewPromise(); - TPromise<void> promise2 = NewPromise(); + TPromise<void> promise1 = NewPromise(); + TPromise<void> promise2 = NewPromise(); - TFuture<void> future = WaitAny(promise1, promise2); - UNIT_ASSERT(!future.HasValue()); + TFuture<void> future = WaitAny(promise1, promise2); + UNIT_ASSERT(!future.HasValue()); - promise1.SetValue(); - UNIT_ASSERT(future.HasValue()); + promise1.SetValue(); + UNIT_ASSERT(future.HasValue()); - promise2.SetValue(); - UNIT_ASSERT(future.HasValue()); - } + promise2.SetValue(); + UNIT_ASSERT(future.HasValue()); + } Y_UNIT_TEST(ShouldStoreTypesWithoutDefaultConstructor) { - // compileability test - struct TRec { - explicit TRec(int) { - } - }; + // compileability test + struct TRec { + explicit TRec(int) { + } + }; - auto promise = NewPromise<TRec>(); - promise.SetValue(TRec(1)); + auto promise = NewPromise<TRec>(); + promise.SetValue(TRec(1)); - auto future = MakeFuture(TRec(1)); - const auto& rec = future.GetValue(); - Y_UNUSED(rec); - } + auto future = MakeFuture(TRec(1)); + const auto& rec = future.GetValue(); + Y_UNUSED(rec); + } Y_UNIT_TEST(ShouldStoreMovableTypes) { - // compileability test - struct TRec : TMoveOnly { - explicit TRec(int) { - } - }; + // compileability test + struct TRec : TMoveOnly { + explicit TRec(int) { + } + }; - auto promise = NewPromise<TRec>(); - promise.SetValue(TRec(1)); + auto promise = NewPromise<TRec>(); + promise.SetValue(TRec(1)); - auto future = MakeFuture(TRec(1)); - const auto& rec = future.GetValue(); - Y_UNUSED(rec); - } + auto future = MakeFuture(TRec(1)); + const auto& rec = future.GetValue(); + Y_UNUSED(rec); + } Y_UNIT_TEST(ShouldMoveMovableTypes) { - // compileability test - struct TRec : TMoveOnly { - explicit TRec(int) { - } - }; + // compileability test + struct TRec : TMoveOnly { + explicit TRec(int) { + } + }; - auto promise = NewPromise<TRec>(); - promise.SetValue(TRec(1)); + auto promise = NewPromise<TRec>(); + promise.SetValue(TRec(1)); - auto future = MakeFuture(TRec(1)); - auto rec = future.ExtractValue(); - Y_UNUSED(rec); - } + auto future = MakeFuture(TRec(1)); + auto rec = future.ExtractValue(); + Y_UNUSED(rec); + } Y_UNIT_TEST(ShouldNotExtractAfterGet) { - TPromise<int> promise = NewPromise<int>(); - promise.SetValue(123); - UNIT_ASSERT(promise.HasValue()); - UNIT_ASSERT_EQUAL(promise.GetValue(), 123); - UNIT_CHECK_GENERATED_EXCEPTION(promise.ExtractValue(), TFutureException); - } + TPromise<int> promise = NewPromise<int>(); + promise.SetValue(123); + UNIT_ASSERT(promise.HasValue()); + UNIT_ASSERT_EQUAL(promise.GetValue(), 123); + UNIT_CHECK_GENERATED_EXCEPTION(promise.ExtractValue(), TFutureException); + } Y_UNIT_TEST(ShouldNotGetAfterExtract) { - TPromise<int> promise = NewPromise<int>(); - promise.SetValue(123); - UNIT_ASSERT(promise.HasValue()); - UNIT_ASSERT_EQUAL(promise.ExtractValue(), 123); - UNIT_CHECK_GENERATED_EXCEPTION(promise.GetValue(), TFutureException); - } + TPromise<int> promise = NewPromise<int>(); + promise.SetValue(123); + UNIT_ASSERT(promise.HasValue()); + UNIT_ASSERT_EQUAL(promise.ExtractValue(), 123); + UNIT_CHECK_GENERATED_EXCEPTION(promise.GetValue(), TFutureException); + } Y_UNIT_TEST(ShouldNotExtractAfterExtract) { - TPromise<int> promise = NewPromise<int>(); - promise.SetValue(123); - UNIT_ASSERT(promise.HasValue()); - UNIT_ASSERT_EQUAL(promise.ExtractValue(), 123); - UNIT_CHECK_GENERATED_EXCEPTION(promise.ExtractValue(), TFutureException); - } + TPromise<int> promise = NewPromise<int>(); + promise.SetValue(123); + UNIT_ASSERT(promise.HasValue()); + UNIT_ASSERT_EQUAL(promise.ExtractValue(), 123); + UNIT_CHECK_GENERATED_EXCEPTION(promise.ExtractValue(), TFutureException); + } Y_UNIT_TEST(ShouldNotExtractFromSharedDefault) { UNIT_CHECK_GENERATED_EXCEPTION(MakeFuture<int>().ExtractValue(), TFutureException); diff --git a/library/cpp/threading/future/legacy_future.h b/library/cpp/threading/future/legacy_future.h index 6f1eabad73..c699aadf5c 100644 --- a/library/cpp/threading/future/legacy_future.h +++ b/library/cpp/threading/future/legacy_future.h @@ -4,79 +4,79 @@ #include "future.h" #include <util/thread/factory.h> - + #include <functional> - + namespace NThreading { template <typename TR, bool IgnoreException> class TLegacyFuture: public IThreadFactory::IThreadAble, TNonCopyable { - public: - typedef TR(TFunctionSignature)(); - using TFunctionObjectType = std::function<TFunctionSignature>; - using TResult = typename TFunctionObjectType::result_type; + public: + typedef TR(TFunctionSignature)(); + using TFunctionObjectType = std::function<TFunctionSignature>; + using TResult = typename TFunctionObjectType::result_type; - private: - TFunctionObjectType Func_; - TPromise<TResult> Result_; + private: + TFunctionObjectType Func_; + TPromise<TResult> Result_; THolder<IThreadFactory::IThread> Thread_; - public: + public: inline TLegacyFuture(const TFunctionObjectType func, IThreadFactory* pool = SystemThreadFactory()) - : Func_(func) - , Result_(NewPromise<TResult>()) - , Thread_(pool->Run(this)) - { - } + : Func_(func) + , Result_(NewPromise<TResult>()) + , Thread_(pool->Run(this)) + { + } - inline ~TLegacyFuture() override { - this->Join(); - } + inline ~TLegacyFuture() override { + this->Join(); + } - inline TResult Get() { - this->Join(); - return Result_.GetValue(); - } + inline TResult Get() { + this->Join(); + return Result_.GetValue(); + } - private: - inline void Join() { - if (Thread_) { - Thread_->Join(); - Thread_.Destroy(); - } + private: + inline void Join() { + if (Thread_) { + Thread_->Join(); + Thread_.Destroy(); + } } - template <typename Result, bool IgnoreException_> - struct TExecutor { - static void SetPromise(TPromise<Result>& promise, const TFunctionObjectType& func) { - if (IgnoreException_) { - try { - promise.SetValue(func()); - } catch (...) { - } - } else { + template <typename Result, bool IgnoreException_> + struct TExecutor { + static void SetPromise(TPromise<Result>& promise, const TFunctionObjectType& func) { + if (IgnoreException_) { + try { + promise.SetValue(func()); + } catch (...) { + } + } else { promise.SetValue(func()); } } - }; + }; - template <bool IgnoreException_> - struct TExecutor<void, IgnoreException_> { - static void SetPromise(TPromise<void>& promise, const TFunctionObjectType& func) { - if (IgnoreException_) { - try { - func(); - promise.SetValue(); - } catch (...) { - } - } else { + template <bool IgnoreException_> + struct TExecutor<void, IgnoreException_> { + static void SetPromise(TPromise<void>& promise, const TFunctionObjectType& func) { + if (IgnoreException_) { + try { + func(); + promise.SetValue(); + } catch (...) { + } + } else { func(); promise.SetValue(); } } - }; - - void DoExecute() override { - TExecutor<TResult, IgnoreException>::SetPromise(Result_, Func_); + }; + + void DoExecute() override { + TExecutor<TResult, IgnoreException>::SetPromise(Result_, Func_); } }; diff --git a/library/cpp/threading/future/legacy_future_ut.cpp b/library/cpp/threading/future/legacy_future_ut.cpp index ff63db1725..96b46ccebf 100644 --- a/library/cpp/threading/future/legacy_future_ut.cpp +++ b/library/cpp/threading/future/legacy_future_ut.cpp @@ -4,69 +4,69 @@ namespace NThreading { Y_UNIT_TEST_SUITE(TLegacyFutureTest) { - int intf() { - return 17; - } + int intf() { + return 17; + } Y_UNIT_TEST(TestIntFunction) { - TLegacyFuture<int> f((&intf)); - UNIT_ASSERT_VALUES_EQUAL(17, f.Get()); - } + TLegacyFuture<int> f((&intf)); + UNIT_ASSERT_VALUES_EQUAL(17, f.Get()); + } - static int r; + static int r; - void voidf() { - r = 18; - } + void voidf() { + r = 18; + } Y_UNIT_TEST(TestVoidFunction) { - r = 0; - TLegacyFuture<> f((&voidf)); - f.Get(); - UNIT_ASSERT_VALUES_EQUAL(18, r); - } + r = 0; + TLegacyFuture<> f((&voidf)); + f.Get(); + UNIT_ASSERT_VALUES_EQUAL(18, r); + } - struct TSampleClass { - int mValue; + struct TSampleClass { + int mValue; - TSampleClass(int value) - : mValue(value) - { - } + TSampleClass(int value) + : mValue(value) + { + } - int Calc() { - return mValue + 1; - } - }; + int Calc() { + return mValue + 1; + } + }; Y_UNIT_TEST(TestMethod) { - TLegacyFuture<int> f11(std::bind(&TSampleClass::Calc, TSampleClass(3))); - UNIT_ASSERT_VALUES_EQUAL(4, f11.Get()); + TLegacyFuture<int> f11(std::bind(&TSampleClass::Calc, TSampleClass(3))); + UNIT_ASSERT_VALUES_EQUAL(4, f11.Get()); TLegacyFuture<int> f12(std::bind(&TSampleClass::Calc, TSampleClass(3)), SystemThreadFactory()); - UNIT_ASSERT_VALUES_EQUAL(4, f12.Get()); + UNIT_ASSERT_VALUES_EQUAL(4, f12.Get()); - TSampleClass c(5); + TSampleClass c(5); - TLegacyFuture<int> f21(std::bind(&TSampleClass::Calc, std::ref(c))); - UNIT_ASSERT_VALUES_EQUAL(6, f21.Get()); + TLegacyFuture<int> f21(std::bind(&TSampleClass::Calc, std::ref(c))); + UNIT_ASSERT_VALUES_EQUAL(6, f21.Get()); TLegacyFuture<int> f22(std::bind(&TSampleClass::Calc, std::ref(c)), SystemThreadFactory()); - UNIT_ASSERT_VALUES_EQUAL(6, f22.Get()); - } + UNIT_ASSERT_VALUES_EQUAL(6, f22.Get()); + } struct TSomeThreadPool: public IThreadFactory {}; Y_UNIT_TEST(TestFunction) { - std::function<int()> f((&intf)); + std::function<int()> f((&intf)); - UNIT_ASSERT_VALUES_EQUAL(17, TLegacyFuture<int>(f).Get()); + UNIT_ASSERT_VALUES_EQUAL(17, TLegacyFuture<int>(f).Get()); UNIT_ASSERT_VALUES_EQUAL(17, TLegacyFuture<int>(f, SystemThreadFactory()).Get()); - if (false) { - TSomeThreadPool* q = nullptr; - TLegacyFuture<int>(f, q); // just check compiles, do not start - } + if (false) { + TSomeThreadPool* q = nullptr; + TLegacyFuture<int>(f, q); // just check compiles, do not start + } } } diff --git a/library/cpp/threading/future/perf/main.cpp b/library/cpp/threading/future/perf/main.cpp index 5a0690af47..c7da5a51f3 100644 --- a/library/cpp/threading/future/perf/main.cpp +++ b/library/cpp/threading/future/perf/main.cpp @@ -7,44 +7,44 @@ using namespace NThreading; template <typename T> -void TestAllocPromise(const NBench::NCpu::TParams& iface) { - for (const auto it : xrange(iface.Iterations())) { +void TestAllocPromise(const NBench::NCpu::TParams& iface) { + for (const auto it : xrange(iface.Iterations())) { Y_UNUSED(it); Y_DO_NOT_OPTIMIZE_AWAY(NewPromise<T>()); } } template <typename T> -TPromise<T> SetPromise(T value) { +TPromise<T> SetPromise(T value) { auto promise = NewPromise<T>(); promise.SetValue(value); return promise; } template <typename T> -void TestSetPromise(const NBench::NCpu::TParams& iface, T value) { - for (const auto it : xrange(iface.Iterations())) { +void TestSetPromise(const NBench::NCpu::TParams& iface, T value) { + for (const auto it : xrange(iface.Iterations())) { Y_UNUSED(it); Y_DO_NOT_OPTIMIZE_AWAY(SetPromise(value)); } } -Y_CPU_BENCHMARK(AllocPromiseVoid, iface) { +Y_CPU_BENCHMARK(AllocPromiseVoid, iface) { TestAllocPromise<void>(iface); } -Y_CPU_BENCHMARK(AllocPromiseUI64, iface) { +Y_CPU_BENCHMARK(AllocPromiseUI64, iface) { TestAllocPromise<ui64>(iface); } -Y_CPU_BENCHMARK(AllocPromiseStroka, iface) { +Y_CPU_BENCHMARK(AllocPromiseStroka, iface) { TestAllocPromise<TString>(iface); } -Y_CPU_BENCHMARK(SetPromiseUI64, iface) { +Y_CPU_BENCHMARK(SetPromiseUI64, iface) { TestSetPromise<ui64>(iface, 1234567890ull); } -Y_CPU_BENCHMARK(SetPromiseStroka, iface) { +Y_CPU_BENCHMARK(SetPromiseStroka, iface) { TestSetPromise<TString>(iface, "test test test"); } diff --git a/library/cpp/threading/future/wait/wait-inl.h b/library/cpp/threading/future/wait/wait-inl.h index 2753d5446c..9d056ff777 100644 --- a/library/cpp/threading/future/wait/wait-inl.h +++ b/library/cpp/threading/future/wait/wait-inl.h @@ -2,10 +2,10 @@ #if !defined(INCLUDE_FUTURE_INL_H) #error "you should never include wait-inl.h directly" -#endif // INCLUDE_FUTURE_INL_H +#endif // INCLUDE_FUTURE_INL_H namespace NThreading { - namespace NImpl { + namespace NImpl { template <typename TContainer> TVector<TFuture<void>> ToVoidFutures(const TContainer& futures) { TVector<TFuture<void>> voidFutures; @@ -17,19 +17,19 @@ namespace NThreading { return voidFutures; } - } + } template <typename TContainer> [[nodiscard]] NImpl::EnableGenericWait<TContainer> WaitAll(const TContainer& futures) { return WaitAll(NImpl::ToVoidFutures(futures)); } - template <typename TContainer> + template <typename TContainer> [[nodiscard]] NImpl::EnableGenericWait<TContainer> WaitExceptionOrAll(const TContainer& futures) { return WaitExceptionOrAll(NImpl::ToVoidFutures(futures)); } - template <typename TContainer> + template <typename TContainer> [[nodiscard]] NImpl::EnableGenericWait<TContainer> WaitAny(const TContainer& futures) { return WaitAny(NImpl::ToVoidFutures(futures)); } diff --git a/library/cpp/threading/future/wait/wait.cpp b/library/cpp/threading/future/wait/wait.cpp index a173833a7f..e0a1c3bbd3 100644 --- a/library/cpp/threading/future/wait/wait.cpp +++ b/library/cpp/threading/future/wait/wait.cpp @@ -31,13 +31,13 @@ namespace NThreading { TWaitGroup<WaitPolicy> wg; for (const auto& fut : futures) { wg.Add(fut); - } + } return std::move(wg).Finish(); } - } + } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// TFuture<void> WaitAll(const TFuture<void>& f1) { return WaitGeneric<TWaitPolicy::TAll>(f1); @@ -56,25 +56,25 @@ namespace NThreading { TFuture<void> WaitExceptionOrAll(const TFuture<void>& f1) { return WaitGeneric<TWaitPolicy::TExceptionOrAll>(f1); - } + } TFuture<void> WaitExceptionOrAll(const TFuture<void>& f1, const TFuture<void>& f2) { return WaitGeneric<TWaitPolicy::TExceptionOrAll>(f1, f2); - } + } TFuture<void> WaitExceptionOrAll(TArrayRef<const TFuture<void>> futures) { return WaitGeneric<TWaitPolicy::TExceptionOrAll>(futures); } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// TFuture<void> WaitAny(const TFuture<void>& f1) { return WaitGeneric<TWaitPolicy::TAny>(f1); - } + } TFuture<void> WaitAny(const TFuture<void>& f1, const TFuture<void>& f2) { return WaitGeneric<TWaitPolicy::TAny>(f1, f2); - } + } TFuture<void> WaitAny(TArrayRef<const TFuture<void>> futures) { return WaitGeneric<TWaitPolicy::TAny>(futures); diff --git a/library/cpp/threading/future/wait/wait.h b/library/cpp/threading/future/wait/wait.h index 6ff7d57baa..60ec5b6a63 100644 --- a/library/cpp/threading/future/wait/wait.h +++ b/library/cpp/threading/future/wait/wait.h @@ -25,16 +25,16 @@ namespace NThreading { [[nodiscard]] TFuture<void> WaitExceptionOrAll(const TFuture<void>& f1); [[nodiscard]] TFuture<void> WaitExceptionOrAll(const TFuture<void>& f1, const TFuture<void>& f2); [[nodiscard]] TFuture<void> WaitExceptionOrAll(TArrayRef<const TFuture<void>> futures); - template <typename TContainer> + template <typename TContainer> [[nodiscard]] NImpl::EnableGenericWait<TContainer> WaitExceptionOrAll(const TContainer& futures); - // waits for any future + // waits for any future [[nodiscard]] TFuture<void> WaitAny(const TFuture<void>& f1); [[nodiscard]] TFuture<void> WaitAny(const TFuture<void>& f1, const TFuture<void>& f2); [[nodiscard]] TFuture<void> WaitAny(TArrayRef<const TFuture<void>> futures); - template <typename TContainer> + template <typename TContainer> [[nodiscard]] NImpl::EnableGenericWait<TContainer> WaitAny(const TContainer& futures); -} +} #define INCLUDE_FUTURE_INL_H #include "wait-inl.h" diff --git a/library/cpp/threading/light_rw_lock/bench/lightrwlock_test.cpp b/library/cpp/threading/light_rw_lock/bench/lightrwlock_test.cpp index c3027ea544..4faa53fc2a 100644 --- a/library/cpp/threading/light_rw_lock/bench/lightrwlock_test.cpp +++ b/library/cpp/threading/light_rw_lock/bench/lightrwlock_test.cpp @@ -28,7 +28,7 @@ static int Y_FORCE_INLINE AtomicFetchAdd(volatile int& item, int value) { } #elif defined(__GNUC__) #else -#error unsupported platform +#error unsupported platform #endif class TPosixRWLock { diff --git a/library/cpp/threading/light_rw_lock/lightrwlock.cpp b/library/cpp/threading/light_rw_lock/lightrwlock.cpp index fbb63fd47f..107c8ec3bf 100644 --- a/library/cpp/threading/light_rw_lock/lightrwlock.cpp +++ b/library/cpp/threading/light_rw_lock/lightrwlock.cpp @@ -79,7 +79,7 @@ void TLightRWLock::WaitForUntrappedAndAcquireRead() { } skip_lock_try: - if (AtomicLoad(UnshareFutex_) && (AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) { + if (AtomicLoad(UnshareFutex_) && (AtomicLoad(Counter_) & 0x7FFFFFFF) == 0) { SequenceStore(UnshareFutex_, 0); FutexWake(UnshareFutex_, 1); } diff --git a/library/cpp/threading/light_rw_lock/lightrwlock.h b/library/cpp/threading/light_rw_lock/lightrwlock.h index 931a1817bc..4c648fc2a3 100644 --- a/library/cpp/threading/light_rw_lock/lightrwlock.h +++ b/library/cpp/threading/light_rw_lock/lightrwlock.h @@ -1,5 +1,5 @@ #pragma once - + #include <util/system/rwlock.h> #include <util/system/sanitizers.h> @@ -34,55 +34,55 @@ #include <errno.h> namespace NS_LightRWLock { - static int Y_FORCE_INLINE AtomicFetchAdd(volatile int& item, int value) { - return __atomic_fetch_add(&item, value, __ATOMIC_SEQ_CST); - } + static int Y_FORCE_INLINE AtomicFetchAdd(volatile int& item, int value) { + return __atomic_fetch_add(&item, value, __ATOMIC_SEQ_CST); + } #if defined(_x86_64_) || defined(_i386_) - static char Y_FORCE_INLINE AtomicSetBit(volatile int& item, unsigned bit) { - char ret; - __asm__ __volatile__( - "lock bts %2,%0\n" - "setc %1\n" - : "+m"(item), "=rm"(ret) - : "r"(bit) - : "cc"); + static char Y_FORCE_INLINE AtomicSetBit(volatile int& item, unsigned bit) { + char ret; + __asm__ __volatile__( + "lock bts %2,%0\n" + "setc %1\n" + : "+m"(item), "=rm"(ret) + : "r"(bit) + : "cc"); // msan doesn't treat ret as initialized NSan::Unpoison(&ret, sizeof(ret)); - return ret; - } + return ret; + } - static char Y_FORCE_INLINE AtomicClearBit(volatile int& item, unsigned bit) { - char ret; - __asm__ __volatile__( - "lock btc %2,%0\n" - "setc %1\n" - : "+m"(item), "=rm"(ret) - : "r"(bit) - : "cc"); + static char Y_FORCE_INLINE AtomicClearBit(volatile int& item, unsigned bit) { + char ret; + __asm__ __volatile__( + "lock btc %2,%0\n" + "setc %1\n" + : "+m"(item), "=rm"(ret) + : "r"(bit) + : "cc"); // msan doesn't treat ret as initialized NSan::Unpoison(&ret, sizeof(ret)); - return ret; - } + return ret; + } #else - static char Y_FORCE_INLINE AtomicSetBit(volatile int& item, unsigned bit) { - int prev = __atomic_fetch_or(&item, 1 << bit, __ATOMIC_SEQ_CST); - return (prev & (1 << bit)) != 0 ? 1 : 0; - } + static char Y_FORCE_INLINE AtomicSetBit(volatile int& item, unsigned bit) { + int prev = __atomic_fetch_or(&item, 1 << bit, __ATOMIC_SEQ_CST); + return (prev & (1 << bit)) != 0 ? 1 : 0; + } - static char Y_FORCE_INLINE - AtomicClearBit(volatile int& item, unsigned bit) { - int prev = __atomic_fetch_and(&item, ~(1 << bit), __ATOMIC_SEQ_CST); - return (prev & (1 << bit)) != 0 ? 1 : 0; - } + static char Y_FORCE_INLINE + AtomicClearBit(volatile int& item, unsigned bit) { + int prev = __atomic_fetch_and(&item, ~(1 << bit), __ATOMIC_SEQ_CST); + return (prev & (1 << bit)) != 0 ? 1 : 0; + } #endif #if defined(_x86_64_) || defined(_i386_) || defined (__aarch64__) || defined (__powerpc64__) @@ -100,42 +100,42 @@ namespace NS_LightRWLock { #endif - template <typename TInt> - static void Y_FORCE_INLINE AtomicStore(volatile TInt& var, TInt value) { - __atomic_store_n(&var, value, __ATOMIC_RELEASE); - } - - template <typename TInt> - static void Y_FORCE_INLINE SequenceStore(volatile TInt& var, TInt value) { - __atomic_store_n(&var, value, __ATOMIC_SEQ_CST); - } - - template <typename TInt> - static TInt Y_FORCE_INLINE AtomicLoad(const volatile TInt& var) { - return __atomic_load_n(&var, __ATOMIC_ACQUIRE); - } - - static void Y_FORCE_INLINE FutexWait(volatile int& fvar, int value) { - for (;;) { - int result = - syscall(SYS_futex, &fvar, FUTEX_WAIT_PRIVATE, value, NULL, NULL, 0); - if (Y_UNLIKELY(result == -1)) { - if (errno == EWOULDBLOCK) - return; - if (errno == EINTR) - continue; - Y_FAIL("futex error"); - } + template <typename TInt> + static void Y_FORCE_INLINE AtomicStore(volatile TInt& var, TInt value) { + __atomic_store_n(&var, value, __ATOMIC_RELEASE); + } + + template <typename TInt> + static void Y_FORCE_INLINE SequenceStore(volatile TInt& var, TInt value) { + __atomic_store_n(&var, value, __ATOMIC_SEQ_CST); + } + + template <typename TInt> + static TInt Y_FORCE_INLINE AtomicLoad(const volatile TInt& var) { + return __atomic_load_n(&var, __ATOMIC_ACQUIRE); + } + + static void Y_FORCE_INLINE FutexWait(volatile int& fvar, int value) { + for (;;) { + int result = + syscall(SYS_futex, &fvar, FUTEX_WAIT_PRIVATE, value, NULL, NULL, 0); + if (Y_UNLIKELY(result == -1)) { + if (errno == EWOULDBLOCK) + return; + if (errno == EINTR) + continue; + Y_FAIL("futex error"); + } } } - static void Y_FORCE_INLINE FutexWake(volatile int& fvar, int amount) { - const int result = - syscall(SYS_futex, &fvar, FUTEX_WAKE_PRIVATE, amount, NULL, NULL, 0); - if (Y_UNLIKELY(result == -1)) - Y_FAIL("futex error"); - } - + static void Y_FORCE_INLINE FutexWake(volatile int& fvar, int amount) { + const int result = + syscall(SYS_futex, &fvar, FUTEX_WAKE_PRIVATE, amount, NULL, NULL, 0); + if (Y_UNLIKELY(result == -1)) + Y_FAIL("futex error"); + } + } class alignas(64) TLightRWLock { @@ -145,8 +145,8 @@ public: , TrappedFutex_(0) , UnshareFutex_(0) , SpinCount_(spinCount) - { - } + { + } TLightRWLock(const TLightRWLock&) = delete; void operator=(const TLightRWLock&) = delete; @@ -208,10 +208,10 @@ private: class TLightRWLock: public TRWMutex { public: - TLightRWLock() { - } - TLightRWLock(ui32) { - } + TLightRWLock() { + } + TLightRWLock(ui32) { + } }; #endif diff --git a/library/cpp/threading/local_executor/local_executor.cpp b/library/cpp/threading/local_executor/local_executor.cpp index 1d3fbb4bf4..f2b825681f 100644 --- a/library/cpp/threading/local_executor/local_executor.cpp +++ b/library/cpp/threading/local_executor/local_executor.cpp @@ -40,7 +40,7 @@ namespace { NPar::TLocallyExecutableFunction Exec; int FirstId, LastId; TVector<NThreading::TPromise<void>> Promises; - + public: TFunctionWrapperWithPromise(NPar::TLocallyExecutableFunction exec, int firstId, int lastId) : Exec(std::move(exec)) diff --git a/library/cpp/threading/local_executor/local_executor.h b/library/cpp/threading/local_executor/local_executor.h index c1c824f67c..33476722b8 100644 --- a/library/cpp/threading/local_executor/local_executor.h +++ b/library/cpp/threading/local_executor/local_executor.h @@ -5,7 +5,7 @@ #include <util/generic/cast.h> #include <util/generic/fwd.h> #include <util/generic/noncopyable.h> -#include <util/generic/ptr.h> +#include <util/generic/ptr.h> #include <util/generic/singleton.h> #include <util/generic/ymath.h> @@ -135,9 +135,9 @@ namespace NPar { // TVector<NThreading::TFuture<void>> ExecRangeWithFutures(TLocallyExecutableFunction exec, int firstId, int lastId, int flags); - template <typename TBody> + template <typename TBody> static inline auto BlockedLoopBody(const TExecRangeParams& params, const TBody& body) { - return [=](int blockId) { + return [=](int blockId) { const int blockFirstId = params.FirstId + blockId * params.GetBlockSize(); const int blockLastId = Min(params.LastId, blockFirstId + params.GetBlockSize()); for (int i = blockFirstId; i < blockLastId; ++i) { diff --git a/library/cpp/threading/local_executor/ut/local_executor_ut.cpp b/library/cpp/threading/local_executor/ut/local_executor_ut.cpp index ac5737717c..ccc833c1b9 100644 --- a/library/cpp/threading/local_executor/ut/local_executor_ut.cpp +++ b/library/cpp/threading/local_executor/ut/local_executor_ut.cpp @@ -15,315 +15,315 @@ static const int DefaultThreadsCount = 41; static const int DefaultRangeSize = 999; Y_UNIT_TEST_SUITE(ExecRangeWithFutures){ - bool AllOf(const TVector<int>& vec, int value){ + bool AllOf(const TVector<int>& vec, int value){ return AllOf(vec, [value](int element) { return value == element; }); -} - -void AsyncRunAndWaitFuturesReady(int rangeSize, int threads) { - TLocalExecutor localExecutor; - localExecutor.RunAdditionalThreads(threads); - TAtomic signal = 0; - TVector<int> data(rangeSize, 0); - TVector<NThreading::TFuture<void>> futures = localExecutor.ExecRangeWithFutures([&signal, &data](int i) { - UNIT_ASSERT(data[i] == 0); - while (AtomicGet(signal) == 0) - ; - data[i] += 1; - }, - 0, rangeSize, TLocalExecutor::HIGH_PRIORITY); - UNIT_ASSERT(AllOf(data, 0)); - for (auto& future : futures) - UNIT_ASSERT(!future.HasValue()); - AtomicSet(signal, 1); - for (auto& future : futures) { - future.GetValueSync(); +} + +void AsyncRunAndWaitFuturesReady(int rangeSize, int threads) { + TLocalExecutor localExecutor; + localExecutor.RunAdditionalThreads(threads); + TAtomic signal = 0; + TVector<int> data(rangeSize, 0); + TVector<NThreading::TFuture<void>> futures = localExecutor.ExecRangeWithFutures([&signal, &data](int i) { + UNIT_ASSERT(data[i] == 0); + while (AtomicGet(signal) == 0) + ; + data[i] += 1; + }, + 0, rangeSize, TLocalExecutor::HIGH_PRIORITY); + UNIT_ASSERT(AllOf(data, 0)); + for (auto& future : futures) + UNIT_ASSERT(!future.HasValue()); + AtomicSet(signal, 1); + for (auto& future : futures) { + future.GetValueSync(); } - UNIT_ASSERT(AllOf(data, 1)); -} + UNIT_ASSERT(AllOf(data, 1)); +} Y_UNIT_TEST(AsyncRunRangeAndWaitFuturesReady) { - AsyncRunAndWaitFuturesReady(DefaultRangeSize, DefaultThreadsCount); -} + AsyncRunAndWaitFuturesReady(DefaultRangeSize, DefaultThreadsCount); +} Y_UNIT_TEST(AsyncRunOneTaskAndWaitFuturesReady) { - AsyncRunAndWaitFuturesReady(1, DefaultThreadsCount); -} + AsyncRunAndWaitFuturesReady(1, DefaultThreadsCount); +} Y_UNIT_TEST(AsyncRunRangeAndWaitFuturesReadyOneExtraThread) { - AsyncRunAndWaitFuturesReady(DefaultRangeSize, 1); -} + AsyncRunAndWaitFuturesReady(DefaultRangeSize, 1); +} Y_UNIT_TEST(AsyncRunOneThreadAndWaitFuturesReadyOneExtraThread) { - AsyncRunAndWaitFuturesReady(1, 1); -} + AsyncRunAndWaitFuturesReady(1, 1); +} Y_UNIT_TEST(AsyncRunTwoRangesAndWaitFuturesReady) { - TLocalExecutor localExecutor; - localExecutor.RunAdditionalThreads(DefaultThreadsCount); - TAtomic signal = 0; - TVector<int> data1(DefaultRangeSize, 0); - TVector<NThreading::TFuture<void>> futures1 = localExecutor.ExecRangeWithFutures([&signal, &data1](int i) { - UNIT_ASSERT(data1[i] == 0); - while (AtomicGet(signal) == 0) - ; - data1[i] += 1; - }, - 0, DefaultRangeSize, TLocalExecutor::HIGH_PRIORITY); - TVector<int> data2(DefaultRangeSize, 0); - TVector<NThreading::TFuture<void>> futures2 = localExecutor.ExecRangeWithFutures([&signal, &data2](int i) { - UNIT_ASSERT(data2[i] == 0); - while (AtomicGet(signal) == 0) - ; - data2[i] += 2; - }, - 0, DefaultRangeSize, TLocalExecutor::HIGH_PRIORITY); - UNIT_ASSERT(AllOf(data1, 0)); - UNIT_ASSERT(AllOf(data2, 0)); - AtomicSet(signal, 1); - for (int i = 0; i < DefaultRangeSize; ++i) { - futures1[i].GetValueSync(); - futures2[i].GetValueSync(); + TLocalExecutor localExecutor; + localExecutor.RunAdditionalThreads(DefaultThreadsCount); + TAtomic signal = 0; + TVector<int> data1(DefaultRangeSize, 0); + TVector<NThreading::TFuture<void>> futures1 = localExecutor.ExecRangeWithFutures([&signal, &data1](int i) { + UNIT_ASSERT(data1[i] == 0); + while (AtomicGet(signal) == 0) + ; + data1[i] += 1; + }, + 0, DefaultRangeSize, TLocalExecutor::HIGH_PRIORITY); + TVector<int> data2(DefaultRangeSize, 0); + TVector<NThreading::TFuture<void>> futures2 = localExecutor.ExecRangeWithFutures([&signal, &data2](int i) { + UNIT_ASSERT(data2[i] == 0); + while (AtomicGet(signal) == 0) + ; + data2[i] += 2; + }, + 0, DefaultRangeSize, TLocalExecutor::HIGH_PRIORITY); + UNIT_ASSERT(AllOf(data1, 0)); + UNIT_ASSERT(AllOf(data2, 0)); + AtomicSet(signal, 1); + for (int i = 0; i < DefaultRangeSize; ++i) { + futures1[i].GetValueSync(); + futures2[i].GetValueSync(); } - UNIT_ASSERT(AllOf(data1, 1)); - UNIT_ASSERT(AllOf(data2, 2)); -} - -void AsyncRunRangeAndWaitExceptions(int rangeSize, int threadsCount) { - TLocalExecutor localExecutor; - localExecutor.RunAdditionalThreads(threadsCount); - TAtomic signal = 0; - TVector<int> data(rangeSize, 0); - TVector<NThreading::TFuture<void>> futures = localExecutor.ExecRangeWithFutures([&signal, &data](int i) { - UNIT_ASSERT(data[i] == 0); - while (AtomicGet(signal) == 0) - ; - data[i] += 1; - throw 10000 + i; - }, - 0, rangeSize, TLocalExecutor::HIGH_PRIORITY); - UNIT_ASSERT(AllOf(data, 0)); - UNIT_ASSERT(futures.ysize() == rangeSize); - AtomicSet(signal, 1); - int exceptionsCaught = 0; - for (int i = 0; i < rangeSize; ++i) { - try { - futures[i].GetValueSync(); - } catch (int& e) { - if (e == 10000 + i) { - ++exceptionsCaught; + UNIT_ASSERT(AllOf(data1, 1)); + UNIT_ASSERT(AllOf(data2, 2)); +} + +void AsyncRunRangeAndWaitExceptions(int rangeSize, int threadsCount) { + TLocalExecutor localExecutor; + localExecutor.RunAdditionalThreads(threadsCount); + TAtomic signal = 0; + TVector<int> data(rangeSize, 0); + TVector<NThreading::TFuture<void>> futures = localExecutor.ExecRangeWithFutures([&signal, &data](int i) { + UNIT_ASSERT(data[i] == 0); + while (AtomicGet(signal) == 0) + ; + data[i] += 1; + throw 10000 + i; + }, + 0, rangeSize, TLocalExecutor::HIGH_PRIORITY); + UNIT_ASSERT(AllOf(data, 0)); + UNIT_ASSERT(futures.ysize() == rangeSize); + AtomicSet(signal, 1); + int exceptionsCaught = 0; + for (int i = 0; i < rangeSize; ++i) { + try { + futures[i].GetValueSync(); + } catch (int& e) { + if (e == 10000 + i) { + ++exceptionsCaught; } } } - UNIT_ASSERT(exceptionsCaught == rangeSize); - UNIT_ASSERT(AllOf(data, 1)); -} + UNIT_ASSERT(exceptionsCaught == rangeSize); + UNIT_ASSERT(AllOf(data, 1)); +} Y_UNIT_TEST(AsyncRunRangeAndWaitExceptions) { - AsyncRunRangeAndWaitExceptions(DefaultRangeSize, DefaultThreadsCount); -} + AsyncRunRangeAndWaitExceptions(DefaultRangeSize, DefaultThreadsCount); +} Y_UNIT_TEST(AsyncRunOneTaskAndWaitExceptions) { - AsyncRunRangeAndWaitExceptions(1, DefaultThreadsCount); -} + AsyncRunRangeAndWaitExceptions(1, DefaultThreadsCount); +} Y_UNIT_TEST(AsyncRunRangeAndWaitExceptionsOneExtraThread) { - AsyncRunRangeAndWaitExceptions(DefaultRangeSize, 1); -} + AsyncRunRangeAndWaitExceptions(DefaultRangeSize, 1); +} Y_UNIT_TEST(AsyncRunOneTaskAndWaitExceptionsOneExtraThread) { - AsyncRunRangeAndWaitExceptions(1, 1); -} + AsyncRunRangeAndWaitExceptions(1, 1); +} Y_UNIT_TEST(AsyncRunTwoRangesAndWaitExceptions) { - TLocalExecutor localExecutor; - localExecutor.RunAdditionalThreads(DefaultThreadsCount); - TAtomic signal = 0; - TVector<int> data1(DefaultRangeSize, 0); - TVector<NThreading::TFuture<void>> futures1 = localExecutor.ExecRangeWithFutures([&signal, &data1](int i) { - UNIT_ASSERT(data1[i] == 0); - while (AtomicGet(signal) == 0) - ; - data1[i] += 1; - throw 15000 + i; - }, - 0, DefaultRangeSize, TLocalExecutor::LOW_PRIORITY); - TVector<int> data2(DefaultRangeSize, 0); - TVector<NThreading::TFuture<void>> futures2 = localExecutor.ExecRangeWithFutures([&signal, &data2](int i) { - UNIT_ASSERT(data2[i] == 0); - while (AtomicGet(signal) == 0) - ; - data2[i] += 2; - throw 16000 + i; - }, - 0, DefaultRangeSize, TLocalExecutor::HIGH_PRIORITY); - - UNIT_ASSERT(AllOf(data1, 0)); - UNIT_ASSERT(AllOf(data2, 0)); - UNIT_ASSERT(futures1.size() == DefaultRangeSize); - UNIT_ASSERT(futures2.size() == DefaultRangeSize); - AtomicSet(signal, 1); - int exceptionsCaught = 0; - for (int i = 0; i < DefaultRangeSize; ++i) { - try { - futures1[i].GetValueSync(); - } catch (int& e) { - if (e == 15000 + i) { - ++exceptionsCaught; + TLocalExecutor localExecutor; + localExecutor.RunAdditionalThreads(DefaultThreadsCount); + TAtomic signal = 0; + TVector<int> data1(DefaultRangeSize, 0); + TVector<NThreading::TFuture<void>> futures1 = localExecutor.ExecRangeWithFutures([&signal, &data1](int i) { + UNIT_ASSERT(data1[i] == 0); + while (AtomicGet(signal) == 0) + ; + data1[i] += 1; + throw 15000 + i; + }, + 0, DefaultRangeSize, TLocalExecutor::LOW_PRIORITY); + TVector<int> data2(DefaultRangeSize, 0); + TVector<NThreading::TFuture<void>> futures2 = localExecutor.ExecRangeWithFutures([&signal, &data2](int i) { + UNIT_ASSERT(data2[i] == 0); + while (AtomicGet(signal) == 0) + ; + data2[i] += 2; + throw 16000 + i; + }, + 0, DefaultRangeSize, TLocalExecutor::HIGH_PRIORITY); + + UNIT_ASSERT(AllOf(data1, 0)); + UNIT_ASSERT(AllOf(data2, 0)); + UNIT_ASSERT(futures1.size() == DefaultRangeSize); + UNIT_ASSERT(futures2.size() == DefaultRangeSize); + AtomicSet(signal, 1); + int exceptionsCaught = 0; + for (int i = 0; i < DefaultRangeSize; ++i) { + try { + futures1[i].GetValueSync(); + } catch (int& e) { + if (e == 15000 + i) { + ++exceptionsCaught; } - } - try { - futures2[i].GetValueSync(); - } catch (int& e) { - if (e == 16000 + i) { - ++exceptionsCaught; + } + try { + futures2[i].GetValueSync(); + } catch (int& e) { + if (e == 16000 + i) { + ++exceptionsCaught; } } } - UNIT_ASSERT(exceptionsCaught == 2 * DefaultRangeSize); - UNIT_ASSERT(AllOf(data1, 1)); - UNIT_ASSERT(AllOf(data2, 2)); -} - -void RunRangeAndCheckExceptionsWithWaitComplete(int rangeSize, int threadsCount) { - TLocalExecutor localExecutor; - localExecutor.RunAdditionalThreads(threadsCount); - TVector<int> data(rangeSize, 0); - TVector<NThreading::TFuture<void>> futures = localExecutor.ExecRangeWithFutures([&data](int i) { - UNIT_ASSERT(data[i] == 0); - data[i] += 1; - throw 30000 + i; - }, - 0, rangeSize, TLocalExecutor::EFlags::WAIT_COMPLETE); - UNIT_ASSERT(AllOf(data, 1)); - int exceptionsCaught = 0; - for (int i = 0; i < rangeSize; ++i) { - try { - futures[i].GetValueSync(); - } catch (int& e) { - if (e == 30000 + i) { - ++exceptionsCaught; + UNIT_ASSERT(exceptionsCaught == 2 * DefaultRangeSize); + UNIT_ASSERT(AllOf(data1, 1)); + UNIT_ASSERT(AllOf(data2, 2)); +} + +void RunRangeAndCheckExceptionsWithWaitComplete(int rangeSize, int threadsCount) { + TLocalExecutor localExecutor; + localExecutor.RunAdditionalThreads(threadsCount); + TVector<int> data(rangeSize, 0); + TVector<NThreading::TFuture<void>> futures = localExecutor.ExecRangeWithFutures([&data](int i) { + UNIT_ASSERT(data[i] == 0); + data[i] += 1; + throw 30000 + i; + }, + 0, rangeSize, TLocalExecutor::EFlags::WAIT_COMPLETE); + UNIT_ASSERT(AllOf(data, 1)); + int exceptionsCaught = 0; + for (int i = 0; i < rangeSize; ++i) { + try { + futures[i].GetValueSync(); + } catch (int& e) { + if (e == 30000 + i) { + ++exceptionsCaught; } } } - UNIT_ASSERT(exceptionsCaught == rangeSize); - UNIT_ASSERT(AllOf(data, 1)); -} + UNIT_ASSERT(exceptionsCaught == rangeSize); + UNIT_ASSERT(AllOf(data, 1)); +} Y_UNIT_TEST(RunRangeAndCheckExceptionsWithWaitComplete) { - RunRangeAndCheckExceptionsWithWaitComplete(DefaultRangeSize, DefaultThreadsCount); -} + RunRangeAndCheckExceptionsWithWaitComplete(DefaultRangeSize, DefaultThreadsCount); +} Y_UNIT_TEST(RunOneAndCheckExceptionsWithWaitComplete) { - RunRangeAndCheckExceptionsWithWaitComplete(1, DefaultThreadsCount); -} + RunRangeAndCheckExceptionsWithWaitComplete(1, DefaultThreadsCount); +} Y_UNIT_TEST(RunRangeAndCheckExceptionsWithWaitCompleteOneExtraThread) { - RunRangeAndCheckExceptionsWithWaitComplete(DefaultRangeSize, 1); -} + RunRangeAndCheckExceptionsWithWaitComplete(DefaultRangeSize, 1); +} Y_UNIT_TEST(RunOneAndCheckExceptionsWithWaitCompleteOneExtraThread) { - RunRangeAndCheckExceptionsWithWaitComplete(1, 1); -} + RunRangeAndCheckExceptionsWithWaitComplete(1, 1); +} Y_UNIT_TEST(RunRangeAndCheckExceptionsWithWaitCompleteZeroExtraThreads) { - RunRangeAndCheckExceptionsWithWaitComplete(DefaultRangeSize, 0); -} + RunRangeAndCheckExceptionsWithWaitComplete(DefaultRangeSize, 0); +} Y_UNIT_TEST(RunOneAndCheckExceptionsWithWaitCompleteZeroExtraThreads) { - RunRangeAndCheckExceptionsWithWaitComplete(1, 0); -} -} -; + RunRangeAndCheckExceptionsWithWaitComplete(1, 0); +} +} +; Y_UNIT_TEST_SUITE(ExecRangeWithThrow){ - void RunParallelWhichThrowsTTestException(int rangeStart, int rangeSize, int threadsCount, int flags, TAtomic& processed){ - AtomicSet(processed, 0); -TLocalExecutor localExecutor; -localExecutor.RunAdditionalThreads(threadsCount); -localExecutor.ExecRangeWithThrow([&processed](int) { - AtomicAdd(processed, 1); - throw TTestException(); -}, - rangeStart, rangeStart + rangeSize, flags); -} + void RunParallelWhichThrowsTTestException(int rangeStart, int rangeSize, int threadsCount, int flags, TAtomic& processed){ + AtomicSet(processed, 0); +TLocalExecutor localExecutor; +localExecutor.RunAdditionalThreads(threadsCount); +localExecutor.ExecRangeWithThrow([&processed](int) { + AtomicAdd(processed, 1); + throw TTestException(); +}, + rangeStart, rangeStart + rangeSize, flags); +} Y_UNIT_TEST(RunParallelWhichThrowsTTestException) { - TAtomic processed = 0; - UNIT_ASSERT_EXCEPTION( - RunParallelWhichThrowsTTestException(10, 40, DefaultThreadsCount, - TLocalExecutor::EFlags::WAIT_COMPLETE, processed), - TTestException); - UNIT_ASSERT(AtomicGet(processed) == 40); -} - -void ThrowAndCatchTTestException(int rangeSize, int threadsCount, int flags) { - TAtomic processed = 0; - UNIT_ASSERT_EXCEPTION( - RunParallelWhichThrowsTTestException(0, rangeSize, threadsCount, flags, processed), - TTestException); - UNIT_ASSERT(AtomicGet(processed) == rangeSize); -} + TAtomic processed = 0; + UNIT_ASSERT_EXCEPTION( + RunParallelWhichThrowsTTestException(10, 40, DefaultThreadsCount, + TLocalExecutor::EFlags::WAIT_COMPLETE, processed), + TTestException); + UNIT_ASSERT(AtomicGet(processed) == 40); +} + +void ThrowAndCatchTTestException(int rangeSize, int threadsCount, int flags) { + TAtomic processed = 0; + UNIT_ASSERT_EXCEPTION( + RunParallelWhichThrowsTTestException(0, rangeSize, threadsCount, flags, processed), + TTestException); + UNIT_ASSERT(AtomicGet(processed) == rangeSize); +} Y_UNIT_TEST(ThrowAndCatchTTestExceptionLowPriority) { - ThrowAndCatchTTestException(DefaultRangeSize, DefaultThreadsCount, - TLocalExecutor::EFlags::WAIT_COMPLETE | TLocalExecutor::EFlags::LOW_PRIORITY); -} + ThrowAndCatchTTestException(DefaultRangeSize, DefaultThreadsCount, + TLocalExecutor::EFlags::WAIT_COMPLETE | TLocalExecutor::EFlags::LOW_PRIORITY); +} Y_UNIT_TEST(ThrowAndCatchTTestExceptionMedPriority) { - ThrowAndCatchTTestException(DefaultRangeSize, DefaultThreadsCount, - TLocalExecutor::EFlags::WAIT_COMPLETE | TLocalExecutor::EFlags::MED_PRIORITY); -} + ThrowAndCatchTTestException(DefaultRangeSize, DefaultThreadsCount, + TLocalExecutor::EFlags::WAIT_COMPLETE | TLocalExecutor::EFlags::MED_PRIORITY); +} Y_UNIT_TEST(ThrowAndCatchTTestExceptionHighPriority) { - ThrowAndCatchTTestException(DefaultRangeSize, DefaultThreadsCount, - TLocalExecutor::EFlags::WAIT_COMPLETE | TLocalExecutor::EFlags::HIGH_PRIORITY); -} + ThrowAndCatchTTestException(DefaultRangeSize, DefaultThreadsCount, + TLocalExecutor::EFlags::WAIT_COMPLETE | TLocalExecutor::EFlags::HIGH_PRIORITY); +} Y_UNIT_TEST(ThrowAndCatchTTestExceptionWaitComplete) { - ThrowAndCatchTTestException(DefaultRangeSize, DefaultThreadsCount, - TLocalExecutor::EFlags::WAIT_COMPLETE); -} + ThrowAndCatchTTestException(DefaultRangeSize, DefaultThreadsCount, + TLocalExecutor::EFlags::WAIT_COMPLETE); +} Y_UNIT_TEST(RethrowExeptionSequentialWaitComplete) { - ThrowAndCatchTTestException(DefaultRangeSize, 0, - TLocalExecutor::EFlags::WAIT_COMPLETE); -} + ThrowAndCatchTTestException(DefaultRangeSize, 0, + TLocalExecutor::EFlags::WAIT_COMPLETE); +} Y_UNIT_TEST(RethrowExeptionOneExtraThreadWaitComplete) { - ThrowAndCatchTTestException(DefaultRangeSize, 1, - TLocalExecutor::EFlags::WAIT_COMPLETE); -} - -void ThrowsTTestExceptionFromNested(TLocalExecutor& localExecutor) { - localExecutor.ExecRangeWithThrow([](int) { - throw TTestException(); - }, - 0, 10, TLocalExecutor::EFlags::WAIT_COMPLETE); -} - -void CatchTTestExceptionFromNested(TAtomic& processed1, TAtomic& processed2) { - TLocalExecutor localExecutor; - localExecutor.RunAdditionalThreads(DefaultThreadsCount); - localExecutor.ExecRangeWithThrow([&processed1, &processed2, &localExecutor](int) { - AtomicAdd(processed1, 1); - UNIT_ASSERT_EXCEPTION( - ThrowsTTestExceptionFromNested(localExecutor), - TTestException); - AtomicAdd(processed2, 1); - }, - 0, DefaultRangeSize, TLocalExecutor::EFlags::WAIT_COMPLETE); -} + ThrowAndCatchTTestException(DefaultRangeSize, 1, + TLocalExecutor::EFlags::WAIT_COMPLETE); +} + +void ThrowsTTestExceptionFromNested(TLocalExecutor& localExecutor) { + localExecutor.ExecRangeWithThrow([](int) { + throw TTestException(); + }, + 0, 10, TLocalExecutor::EFlags::WAIT_COMPLETE); +} + +void CatchTTestExceptionFromNested(TAtomic& processed1, TAtomic& processed2) { + TLocalExecutor localExecutor; + localExecutor.RunAdditionalThreads(DefaultThreadsCount); + localExecutor.ExecRangeWithThrow([&processed1, &processed2, &localExecutor](int) { + AtomicAdd(processed1, 1); + UNIT_ASSERT_EXCEPTION( + ThrowsTTestExceptionFromNested(localExecutor), + TTestException); + AtomicAdd(processed2, 1); + }, + 0, DefaultRangeSize, TLocalExecutor::EFlags::WAIT_COMPLETE); +} Y_UNIT_TEST(NestedParallelExceptionsDoNotLeak) { - TAtomic processed1 = 0; - TAtomic processed2 = 0; - UNIT_ASSERT_NO_EXCEPTION( - CatchTTestExceptionFromNested(processed1, processed2)); - UNIT_ASSERT_EQUAL(AtomicGet(processed1), DefaultRangeSize); - UNIT_ASSERT_EQUAL(AtomicGet(processed2), DefaultRangeSize); -} -} -; + TAtomic processed1 = 0; + TAtomic processed2 = 0; + UNIT_ASSERT_NO_EXCEPTION( + CatchTTestExceptionFromNested(processed1, processed2)); + UNIT_ASSERT_EQUAL(AtomicGet(processed1), DefaultRangeSize); + UNIT_ASSERT_EQUAL(AtomicGet(processed2), DefaultRangeSize); +} +} +; Y_UNIT_TEST_SUITE(ExecLargeRangeWithThrow){ diff --git a/library/cpp/threading/poor_man_openmp/thread_helper.cpp b/library/cpp/threading/poor_man_openmp/thread_helper.cpp index 34cb6507b9..b4ec5c7879 100644 --- a/library/cpp/threading/poor_man_openmp/thread_helper.cpp +++ b/library/cpp/threading/poor_man_openmp/thread_helper.cpp @@ -1,7 +1,7 @@ #include "thread_helper.h" - -#include <util/generic/singleton.h> - -TMtpQueueHelper& TMtpQueueHelper::Instance() { - return *Singleton<TMtpQueueHelper>(); -} + +#include <util/generic/singleton.h> + +TMtpQueueHelper& TMtpQueueHelper::Instance() { + return *Singleton<TMtpQueueHelper>(); +} diff --git a/library/cpp/threading/poor_man_openmp/thread_helper.h b/library/cpp/threading/poor_man_openmp/thread_helper.h index 0ecee0590b..1536c186cb 100644 --- a/library/cpp/threading/poor_man_openmp/thread_helper.h +++ b/library/cpp/threading/poor_man_openmp/thread_helper.h @@ -2,17 +2,17 @@ #include <util/thread/pool.h> #include <util/generic/utility.h> -#include <util/generic/yexception.h> +#include <util/generic/yexception.h> #include <util/system/info.h> #include <util/system/atomic.h> #include <util/system/condvar.h> #include <util/system/mutex.h> -#include <util/stream/output.h> +#include <util/stream/output.h> #include <functional> -#include <cstdlib> +#include <cstdlib> -class TMtpQueueHelper { +class TMtpQueueHelper { public: TMtpQueueHelper() { SetThreadCount(NSystemInfo::CachedNumberOfCpus()); @@ -27,79 +27,79 @@ public: ThreadCount = threads; q = CreateThreadPool(ThreadCount); } - - static TMtpQueueHelper& Instance(); - + + static TMtpQueueHelper& Instance(); + private: size_t ThreadCount; TAutoPtr<IThreadPool> q; }; -namespace NYmp { +namespace NYmp { inline void SetThreadCount(size_t threads) { - TMtpQueueHelper::Instance().SetThreadCount(threads); + TMtpQueueHelper::Instance().SetThreadCount(threads); } inline size_t GetThreadCount() { - return TMtpQueueHelper::Instance().GetThreadCount(); + return TMtpQueueHelper::Instance().GetThreadCount(); } - template <typename T> + template <typename T> inline void ParallelForStaticChunk(T begin, T end, size_t chunkSize, std::function<void(T)> func) { - chunkSize = Max<size_t>(chunkSize, 1); - - size_t threadCount = TMtpQueueHelper::Instance().GetThreadCount(); + chunkSize = Max<size_t>(chunkSize, 1); + + size_t threadCount = TMtpQueueHelper::Instance().GetThreadCount(); IThreadPool* queue = TMtpQueueHelper::Instance().Get(); TCondVar cv; TMutex mutex; TAtomic counter = threadCount; - std::exception_ptr err; - - for (size_t i = 0; i < threadCount; ++i) { - queue->SafeAddFunc([&cv, &counter, &mutex, &func, i, begin, end, chunkSize, threadCount, &err]() { - try { - T currentChunkStart = begin + static_cast<decltype(T() - T())>(i * chunkSize); - - while (currentChunkStart < end) { - T currentChunkEnd = Min<T>(end, currentChunkStart + chunkSize); - - for (T val = currentChunkStart; val < currentChunkEnd; ++val) { - func(val); - } - - currentChunkStart += chunkSize * threadCount; - } - } catch (...) { - with_lock (mutex) { - err = std::current_exception(); - } - } - - with_lock (mutex) { - if (AtomicDecrement(counter) == 0) { - //last one - cv.Signal(); + std::exception_ptr err; + + for (size_t i = 0; i < threadCount; ++i) { + queue->SafeAddFunc([&cv, &counter, &mutex, &func, i, begin, end, chunkSize, threadCount, &err]() { + try { + T currentChunkStart = begin + static_cast<decltype(T() - T())>(i * chunkSize); + + while (currentChunkStart < end) { + T currentChunkEnd = Min<T>(end, currentChunkStart + chunkSize); + + for (T val = currentChunkStart; val < currentChunkEnd; ++val) { + func(val); + } + + currentChunkStart += chunkSize * threadCount; } + } catch (...) { + with_lock (mutex) { + err = std::current_exception(); + } } + + with_lock (mutex) { + if (AtomicDecrement(counter) == 0) { + //last one + cv.Signal(); + } + } }); } - - with_lock (mutex) { - while (AtomicGet(counter) > 0) { - cv.WaitI(mutex); - } - } - - if (err) { - std::rethrow_exception(err); + + with_lock (mutex) { + while (AtomicGet(counter) > 0) { + cv.WaitI(mutex); + } } + + if (err) { + std::rethrow_exception(err); + } } - template <typename T> + template <typename T> inline void ParallelForStaticAutoChunk(T begin, T end, std::function<void(T)> func) { - const size_t taskSize = end - begin; - const size_t threadCount = TMtpQueueHelper::Instance().GetThreadCount(); - + const size_t taskSize = end - begin; + const size_t threadCount = TMtpQueueHelper::Instance().GetThreadCount(); + ParallelForStaticChunk(begin, end, (taskSize + threadCount - 1) / threadCount, func); } -} +} diff --git a/library/cpp/threading/poor_man_openmp/thread_helper_ut.cpp b/library/cpp/threading/poor_man_openmp/thread_helper_ut.cpp index 7417636864..79c7a14b5e 100644 --- a/library/cpp/threading/poor_man_openmp/thread_helper_ut.cpp +++ b/library/cpp/threading/poor_man_openmp/thread_helper_ut.cpp @@ -1,26 +1,26 @@ -#include "thread_helper.h" - +#include "thread_helper.h" + #include <library/cpp/testing/unittest/registar.h> - + #include <util/generic/string.h> -#include <util/generic/yexception.h> - +#include <util/generic/yexception.h> + Y_UNIT_TEST_SUITE(TestMP) { Y_UNIT_TEST(TestErr) { - std::function<void(int)> f = [](int x) { - if (x == 5) { - ythrow yexception() << "oops"; - } - }; - + std::function<void(int)> f = [](int x) { + if (x == 5) { + ythrow yexception() << "oops"; + } + }; + TString s; - - try { - NYmp::ParallelForStaticAutoChunk(0, 10, f); - } catch (...) { - s = CurrentExceptionMessage(); - } - - UNIT_ASSERT(s.find("oops") > 0); - } -} + + try { + NYmp::ParallelForStaticAutoChunk(0, 10, f); + } catch (...) { + s = CurrentExceptionMessage(); + } + + UNIT_ASSERT(s.find("oops") > 0); + } +} diff --git a/library/cpp/threading/poor_man_openmp/ut/ya.make b/library/cpp/threading/poor_man_openmp/ut/ya.make index 6d7aa123ed..7305d14b99 100644 --- a/library/cpp/threading/poor_man_openmp/ut/ya.make +++ b/library/cpp/threading/poor_man_openmp/ut/ya.make @@ -1,12 +1,12 @@ UNITTEST_FOR(library/cpp/threading/poor_man_openmp) - + OWNER( pg agorodilov ) - -SRCS( - thread_helper_ut.cpp -) - -END() + +SRCS( + thread_helper_ut.cpp +) + +END() diff --git a/library/cpp/threading/queue/mpsc_htswap.h b/library/cpp/threading/queue/mpsc_htswap.h index c42caa7ac0..420b1e8829 100644 --- a/library/cpp/threading/queue/mpsc_htswap.h +++ b/library/cpp/threading/queue/mpsc_htswap.h @@ -28,8 +28,8 @@ namespace NThreading { namespace NHTSwapPrivate { template <typename T, typename TTuneup> struct TNode - : public TTuneup::TNodeBase, - public TTuneup::template TNodeLayout<TNode<T, TTuneup>, T> { + : public TTuneup::TNodeBase, + public TTuneup::template TNodeLayout<TNode<T, TTuneup>, T> { TNode(const T& item) { this->Next = nullptr; this->Item = item; @@ -60,7 +60,7 @@ namespace NThreading { template <typename T, typename TTuneup> class THTSwapQueueImpl - : protected TTuneup::template TQueueLayout<TNode<T, TTuneup>> { + : protected TTuneup::template TQueueLayout<TNode<T, TTuneup>> { protected: using TTunedNode = TNode<T, TTuneup>; @@ -124,9 +124,9 @@ namespace NThreading { DeclareTuneTypeParam(THTSwapNodeLayout, TNodeLayout); DeclareTuneTypeParam(THTSwapQueueLayout, TQueueLayout); - template <typename T = void*, typename... TParams> + template <typename T = void*, typename... TParams> class THTSwapQueue - : public NHTSwapPrivate::THTSwapQueueImpl<T, - TTune<NHTSwapPrivate::TDefaultTuneup, TParams...>> { + : public NHTSwapPrivate::THTSwapQueueImpl<T, + TTune<NHTSwapPrivate::TDefaultTuneup, TParams...>> { }; } diff --git a/library/cpp/threading/queue/mpsc_intrusive_unordered.h b/library/cpp/threading/queue/mpsc_intrusive_unordered.h index 6ac7537ae9..97e6131dd4 100644 --- a/library/cpp/threading/queue/mpsc_intrusive_unordered.h +++ b/library/cpp/threading/queue/mpsc_intrusive_unordered.h @@ -25,7 +25,7 @@ namespace NThreading { void Push(void* node) noexcept { Push(reinterpret_cast<TIntrusiveNode*>(node)); } - + private: TIntrusiveNode* HeadForCaS = nullptr; TIntrusiveNode* HeadForSwap = nullptr; diff --git a/library/cpp/threading/queue/mpsc_read_as_filled.h b/library/cpp/threading/queue/mpsc_read_as_filled.h index be33ba5a58..621517328e 100644 --- a/library/cpp/threading/queue/mpsc_read_as_filled.h +++ b/library/cpp/threading/queue/mpsc_read_as_filled.h @@ -132,7 +132,7 @@ namespace NThreading { TMsgBunch* volatile NextToken; /* this push can return PUSH_RESULT_BLOCKED */ - inline TPushResult Push(TMsgLink msg, ui64 slot, TAux auxiliary) { + inline TPushResult Push(TMsgLink msg, ui64 slot, TAux auxiliary) { if (Y_UNLIKELY(slot < FirstSlot)) { return PUSH_RESULT_BACKWARD; } @@ -194,7 +194,7 @@ namespace NThreading { // the object could be destroyed after this method inline void SetNextToken(TMsgBunch* next) { AtomicSet(NextToken, next); - if (Y_UNLIKELY(AtomicAdd(Token, RELEASE_SIZE) == RELEASE_SIZE)) { + if (Y_UNLIKELY(AtomicAdd(Token, RELEASE_SIZE) == RELEASE_SIZE)) { Release(this); next->DecrementToken(); } @@ -317,8 +317,8 @@ namespace NThreading { } }; - template <typename TWBucket = TWriteBucket<>, - template <typename, typename...> class TContainer = TDeque> + template <typename TWBucket = TWriteBucket<>, + template <typename, typename...> class TContainer = TDeque> class TReadBucket { public: using TAux = typename TWBucket::TUsingAux; @@ -543,7 +543,7 @@ namespace NThreading { static constexpr ui32 BUNCH_SIZE = DEFAULT_BUNCH_SIZE; using TBunchBase = TEmpty; - template <typename TElem, typename... TRest> + template <typename TElem, typename... TRest> using TContainer = TDeque<TElem, TRest...>; static constexpr bool DeleteItems = true; @@ -556,7 +556,7 @@ namespace NThreading { DeclareTuneContainer(TRaFQueueSkipContainer, TContainer); DeclareTuneValueParam(TRaFQueueDeleteItems, bool, DeleteItems); - template <typename TItem = void, typename... TParams> + template <typename TItem = void, typename... TParams> class TReadAsFilledQueue { private: using TTuned = TTune<NReadAsFilledPrivate::TDefaultParams, TParams...>; @@ -565,7 +565,7 @@ namespace NThreading { using TBunchBase = typename TTuned::TBunchBase; - template <typename TElem, typename... TRest> + template <typename TElem, typename... TRest> using TContainer = typename TTuned::template TContainer<TElem, TRest...>; diff --git a/library/cpp/threading/queue/mpsc_vinfarr_obstructive.h b/library/cpp/threading/queue/mpsc_vinfarr_obstructive.h index 5f91f1b5a8..4c85bef6ec 100644 --- a/library/cpp/threading/queue/mpsc_vinfarr_obstructive.h +++ b/library/cpp/threading/queue/mpsc_vinfarr_obstructive.h @@ -469,7 +469,7 @@ namespace NThreading { DeclareTuneTypeParam(TObstructiveQueueBunchBase, TBunchBase); DeclareTuneTypeParam(TObstructiveQueueAux, TAux); - template <typename TItem = void, typename... TParams> + template <typename TItem = void, typename... TParams> class TObstructiveConsumerAuxQueue { private: using TTuned = @@ -522,7 +522,7 @@ namespace NThreading { template <typename TItem = void, bool DeleteItems = true> class TObstructiveConsumerQueue - : public TObstructiveConsumerAuxQueue<TItem, - TObstructiveQueueDeleteItems<DeleteItems>> { + : public TObstructiveConsumerAuxQueue<TItem, + TObstructiveQueueDeleteItems<DeleteItems>> { }; -} +} diff --git a/library/cpp/threading/queue/queue_ut.cpp b/library/cpp/threading/queue/queue_ut.cpp index 80eca147da..a55f952cbc 100644 --- a/library/cpp/threading/queue/queue_ut.cpp +++ b/library/cpp/threading/queue/queue_ut.cpp @@ -12,7 +12,7 @@ private: UNIT_TEST(Threads2_Push1M_Threads1_Pop2M) UNIT_TEST(Threads4_Push1M_Threads1_Pop4M) UNIT_TEST(Threads8_RndPush100K_Threads8_Queues) - /* + /* UNIT_TEST(Threads24_RndPush100K_Threads24_Queues) UNIT_TEST(Threads24_RndPush100K_Threads8_Queues) UNIT_TEST(Threads24_RndPush100K_Threads4_Queues) diff --git a/library/cpp/threading/queue/tune.h b/library/cpp/threading/queue/tune.h index 50fc3dc17c..1072342620 100644 --- a/library/cpp/threading/queue/tune.h +++ b/library/cpp/threading/queue/tune.h @@ -96,14 +96,14 @@ }; \ } -#define DeclareTuneContainer(TParamName, InternalName) \ - template <template <typename, typename...> class TNewContainer> \ - struct TParamName { \ - template <typename TBase> \ - struct TApply: public TBase { \ - template <typename TElem, typename... TRest> \ - using InternalName = TNewContainer<TElem, TRest...>; \ - }; \ +#define DeclareTuneContainer(TParamName, InternalName) \ + template <template <typename, typename...> class TNewContainer> \ + struct TParamName { \ + template <typename TBase> \ + struct TApply: public TBase { \ + template <typename TElem, typename... TRest> \ + using InternalName = TNewContainer<TElem, TRest...>; \ + }; \ } namespace NTunePrivate { diff --git a/library/cpp/threading/queue/unordered_ut.cpp b/library/cpp/threading/queue/unordered_ut.cpp index a43b7f520e..49ebd4a1cf 100644 --- a/library/cpp/threading/queue/unordered_ut.cpp +++ b/library/cpp/threading/queue/unordered_ut.cpp @@ -59,9 +59,9 @@ public: class TWorker: public ISimpleThread { public: TWorker( - TQueueType* queues_, - ui16 mine, - TAtomic* pushDone) + TQueueType* queues_, + ui16 mine, + TAtomic* pushDone) : Queues(queues_) , MineQueue(mine) , PushDone(pushDone) @@ -132,7 +132,7 @@ public: for (ui32 i = 0; i < COUNT; ++i) { workers[i]->Join(); all.insert(all.begin(), - workers[i]->Received.begin(), workers[i]->Received.end()); + workers[i]->Received.begin(), workers[i]->Received.end()); } std::sort(all.begin(), all.end()); diff --git a/library/cpp/threading/queue/ut_helpers.h b/library/cpp/threading/queue/ut_helpers.h index 2756b52601..b017763794 100644 --- a/library/cpp/threading/queue/ut_helpers.h +++ b/library/cpp/threading/queue/ut_helpers.h @@ -13,11 +13,11 @@ struct TBasicReadAsFilled: public NThreading::TReadAsFilledQueue<> { }; struct TBasicObstructiveConsumer - : public NThreading::TObstructiveConsumerQueue<> { + : public NThreading::TObstructiveConsumerQueue<> { }; struct TBasicMPSCIntrusiveUnordered - : public NThreading::TMPSCIntrusiveUnordered { + : public NThreading::TMPSCIntrusiveUnordered { }; struct TIntrusiveLink: public NThreading::TIntrusiveNode { @@ -30,11 +30,11 @@ struct TMPMCUnorderedRing: public NThreading::TMPMCUnorderedRing { } }; -#define REGISTER_TESTS_FOR_ALL_ORDERED_QUEUES(TestTemplate) \ - UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TBasicHTSwap>); \ - UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TBasicReadAsFilled>); \ +#define REGISTER_TESTS_FOR_ALL_ORDERED_QUEUES(TestTemplate) \ + UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TBasicHTSwap>); \ + UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TBasicReadAsFilled>); \ UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TBasicObstructiveConsumer>) -#define REGISTER_TESTS_FOR_ALL_UNORDERED_QUEUES(TestTemplate) \ +#define REGISTER_TESTS_FOR_ALL_UNORDERED_QUEUES(TestTemplate) \ UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TBasicMPSCIntrusiveUnordered>); \ UNIT_TEST_SUITE_REGISTRATION(TestTemplate<TMPMCUnorderedRing>); diff --git a/library/cpp/threading/skip_list/compare.h b/library/cpp/threading/skip_list/compare.h index ac98b3e1ce..336582a1b8 100644 --- a/library/cpp/threading/skip_list/compare.h +++ b/library/cpp/threading/skip_list/compare.h @@ -4,74 +4,74 @@ #include <util/str_stl.h> namespace NThreading { - namespace NImpl { - Y_HAS_MEMBER(compare); - Y_HAS_MEMBER(Compare); + namespace NImpl { + Y_HAS_MEMBER(compare); + Y_HAS_MEMBER(Compare); template <typename T> - inline int CompareImpl(const T& l, const T& r) { - if (l < r) { - return -1; - } else if (r < l) { - return +1; - } else { - return 0; - } + inline int CompareImpl(const T& l, const T& r) { + if (l < r) { + return -1; + } else if (r < l) { + return +1; + } else { + return 0; + } } - template <bool val> - struct TSmallCompareSelector { - template <typename T> - static inline int Compare(const T& l, const T& r) { - return CompareImpl(l, r); - } - }; + template <bool val> + struct TSmallCompareSelector { + template <typename T> + static inline int Compare(const T& l, const T& r) { + return CompareImpl(l, r); + } + }; - template <> - struct TSmallCompareSelector<true> { - template <typename T> - static inline int Compare(const T& l, const T& r) { - return l.compare(r); - } - }; + template <> + struct TSmallCompareSelector<true> { + template <typename T> + static inline int Compare(const T& l, const T& r) { + return l.compare(r); + } + }; - template <bool val> - struct TBigCompareSelector { - template <typename T> - static inline int Compare(const T& l, const T& r) { + template <bool val> + struct TBigCompareSelector { + template <typename T> + static inline int Compare(const T& l, const T& r) { return TSmallCompareSelector<THascompare<T>::value>::Compare(l, r); - } - }; - - template <> - struct TBigCompareSelector<true> { - template <typename T> - static inline int Compare(const T& l, const T& r) { - return l.Compare(r); - } - }; - + } + }; + + template <> + struct TBigCompareSelector<true> { + template <typename T> + static inline int Compare(const T& l, const T& r) { + return l.Compare(r); + } + }; + template <typename T> struct TCompareSelector: public TBigCompareSelector<THasCompare<T>::value> { - }; - } - - //////////////////////////////////////////////////////////////////////////////// - // Generic compare function + }; + } + //////////////////////////////////////////////////////////////////////////////// + // Generic compare function + template <typename T> - inline int Compare(const T& l, const T& r) { - return NImpl::TCompareSelector<T>::Compare(l, r); - } - - //////////////////////////////////////////////////////////////////////////////// - // Generic compare functor - - template <typename T> - struct TCompare { - inline int operator()(const T& l, const T& r) const { - return Compare(l, r); - } + inline int Compare(const T& l, const T& r) { + return NImpl::TCompareSelector<T>::Compare(l, r); + } + + //////////////////////////////////////////////////////////////////////////////// + // Generic compare functor + + template <typename T> + struct TCompare { + inline int operator()(const T& l, const T& r) const { + return Compare(l, r); + } }; } diff --git a/library/cpp/threading/skip_list/perf/main.cpp b/library/cpp/threading/skip_list/perf/main.cpp index 4ad52049e7..d722d43436 100644 --- a/library/cpp/threading/skip_list/perf/main.cpp +++ b/library/cpp/threading/skip_list/perf/main.cpp @@ -14,345 +14,345 @@ #include <util/system/thread.h> namespace { - using namespace NThreading; + using namespace NThreading; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// IOutputStream& LogInfo() { - return Cerr << TInstant::Now() << " INFO: "; - } + return Cerr << TInstant::Now() << " INFO: "; + } IOutputStream& LogError() { - return Cerr << TInstant::Now() << " ERROR: "; - } - - //////////////////////////////////////////////////////////////////////////////// - - struct TListItem { - TStringBuf Key; - TStringBuf Value; - - TListItem(const TStringBuf& key, const TStringBuf& value) - : Key(key) - , Value(value) - { - } - - int Compare(const TListItem& other) const { - return Key.compare(other.Key); - } - }; - - using TListType = TSkipList<TListItem>; - - //////////////////////////////////////////////////////////////////////////////// - - class TRandomData { - private: - TVector<char> Buffer; - - public: - TRandomData() - : Buffer(1024 * 1024) - { - for (size_t i = 0; i < Buffer.size(); ++i) { - Buffer[i] = RandomNumber<char>(); - } - } - - TStringBuf GetString(size_t len) const { - size_t start = RandomNumber(Buffer.size() - len); - return TStringBuf(&Buffer[start], len); - } - - TStringBuf GetString(size_t min, size_t max) const { - return GetString(min + RandomNumber(max - min)); + return Cerr << TInstant::Now() << " ERROR: "; + } + + //////////////////////////////////////////////////////////////////////////////// + + struct TListItem { + TStringBuf Key; + TStringBuf Value; + + TListItem(const TStringBuf& key, const TStringBuf& value) + : Key(key) + , Value(value) + { + } + + int Compare(const TListItem& other) const { + return Key.compare(other.Key); + } + }; + + using TListType = TSkipList<TListItem>; + + //////////////////////////////////////////////////////////////////////////////// + + class TRandomData { + private: + TVector<char> Buffer; + + public: + TRandomData() + : Buffer(1024 * 1024) + { + for (size_t i = 0; i < Buffer.size(); ++i) { + Buffer[i] = RandomNumber<char>(); + } + } + + TStringBuf GetString(size_t len) const { + size_t start = RandomNumber(Buffer.size() - len); + return TStringBuf(&Buffer[start], len); } - }; - //////////////////////////////////////////////////////////////////////////////// - - class TWorkerThread: public ISimpleThread { - private: - std::function<void()> Func; - TDuration Time; - - public: - TWorkerThread(std::function<void()> func) - : Func(func) - { - } - - TDuration GetTime() const { - return Time; - } - - private: - void* ThreadProc() noexcept override { - TInstant started = TInstant::Now(); - Func(); - Time = TInstant::Now() - started; - return nullptr; - } - }; - - inline TAutoPtr<TWorkerThread> StartThread(std::function<void()> func) { - TAutoPtr<TWorkerThread> thread = new TWorkerThread(func); - thread->Start(); - return thread; + TStringBuf GetString(size_t min, size_t max) const { + return GetString(min + RandomNumber(max - min)); + } + }; + + //////////////////////////////////////////////////////////////////////////////// + + class TWorkerThread: public ISimpleThread { + private: + std::function<void()> Func; + TDuration Time; + + public: + TWorkerThread(std::function<void()> func) + : Func(func) + { + } + + TDuration GetTime() const { + return Time; + } + + private: + void* ThreadProc() noexcept override { + TInstant started = TInstant::Now(); + Func(); + Time = TInstant::Now() - started; + return nullptr; + } + }; + + inline TAutoPtr<TWorkerThread> StartThread(std::function<void()> func) { + TAutoPtr<TWorkerThread> thread = new TWorkerThread(func); + thread->Start(); + return thread; } - //////////////////////////////////////////////////////////////////////////////// - - typedef std::function<void()> TTestFunc; - - struct TTest { - TString Name; - TTestFunc Func; - - TTest() { - } - - TTest(const TString& name, const TTestFunc& func) - : Name(name) - , Func(func) - { - } - }; - - //////////////////////////////////////////////////////////////////////////////// - - class TTestSuite { - private: - size_t Iterations = 1000000; - size_t KeyLen = 10; - size_t ValueLen = 100; - size_t NumReaders = 4; - size_t NumWriters = 1; - size_t BatchSize = 20; - - TMemoryPool MemoryPool; - TListType List; - TMutex Mutex; - TRandomData Random; - - TMap<TCiString, TTest> AllTests; - TVector<TTest> Tests; - - public: - TTestSuite() - : MemoryPool(64 * 1024) - , List(MemoryPool) - { - } - - bool Init(int argc, const char* argv[]) { - TVector<TString> tests; - try { - NLastGetopt::TOpts opts; - opts.AddHelpOption(); - -#define OPTION(opt, x) \ - opts.AddLongOption(opt, #x) \ - .Optional() \ - .DefaultValue(ToString(x)) \ - .StoreResult(&x) // end of OPTION - - OPTION('i', Iterations); - OPTION('k', KeyLen); - OPTION('v', ValueLen); - OPTION('r', NumReaders); - OPTION('w', NumWriters); - OPTION('b', BatchSize); + //////////////////////////////////////////////////////////////////////////////// + + typedef std::function<void()> TTestFunc; + + struct TTest { + TString Name; + TTestFunc Func; + + TTest() { + } + + TTest(const TString& name, const TTestFunc& func) + : Name(name) + , Func(func) + { + } + }; + + //////////////////////////////////////////////////////////////////////////////// + + class TTestSuite { + private: + size_t Iterations = 1000000; + size_t KeyLen = 10; + size_t ValueLen = 100; + size_t NumReaders = 4; + size_t NumWriters = 1; + size_t BatchSize = 20; + + TMemoryPool MemoryPool; + TListType List; + TMutex Mutex; + TRandomData Random; + + TMap<TCiString, TTest> AllTests; + TVector<TTest> Tests; + + public: + TTestSuite() + : MemoryPool(64 * 1024) + , List(MemoryPool) + { + } + + bool Init(int argc, const char* argv[]) { + TVector<TString> tests; + try { + NLastGetopt::TOpts opts; + opts.AddHelpOption(); + +#define OPTION(opt, x) \ + opts.AddLongOption(opt, #x) \ + .Optional() \ + .DefaultValue(ToString(x)) \ + .StoreResult(&x) // end of OPTION + + OPTION('i', Iterations); + OPTION('k', KeyLen); + OPTION('v', ValueLen); + OPTION('r', NumReaders); + OPTION('w', NumWriters); + OPTION('b', BatchSize); #undef OPTION - NLastGetopt::TOptsParseResultException optsRes(&opts, argc, argv); - for (const auto& opt : opts.Opts_) { - const NLastGetopt::TOptParseResult* r = optsRes.FindOptParseResult(opt.Get(), true); - if (r) { - LogInfo() << "[-" << opt->GetChar() << "] " << opt->GetName() << ": " << r->Back() << Endl; - } + NLastGetopt::TOptsParseResultException optsRes(&opts, argc, argv); + for (const auto& opt : opts.Opts_) { + const NLastGetopt::TOptParseResult* r = optsRes.FindOptParseResult(opt.Get(), true); + if (r) { + LogInfo() << "[-" << opt->GetChar() << "] " << opt->GetName() << ": " << r->Back() << Endl; + } } - tests = optsRes.GetFreeArgs(); - } catch (...) { - LogError() << CurrentExceptionMessage() << Endl; - return false; + tests = optsRes.GetFreeArgs(); + } catch (...) { + LogError() << CurrentExceptionMessage() << Endl; + return false; } #define TEST(type) \ - AddTest(#type, std::bind(&TTestSuite::Y_CAT(TEST_, type), this)) // end of TEST + AddTest(#type, std::bind(&TTestSuite::Y_CAT(TEST_, type), this)) // end of TEST - TEST(Clear); - TEST(InsertRandom); - TEST(InsertSequential); - TEST(InsertSequentialSimple); - TEST(LookupRandom); - TEST(Concurrent); + TEST(Clear); + TEST(InsertRandom); + TEST(InsertSequential); + TEST(InsertSequentialSimple); + TEST(LookupRandom); + TEST(Concurrent); #undef TEST - if (tests.empty()) { - LogError() << "no tests specified, choose from: " << PrintTests() << Endl; + if (tests.empty()) { + LogError() << "no tests specified, choose from: " << PrintTests() << Endl; return false; } - - for (size_t i = 0; i < tests.size(); ++i) { + + for (size_t i = 0; i < tests.size(); ++i) { if (!AllTests.contains(tests[i])) { - LogError() << "unknown test name: " << tests[i] << Endl; - return false; - } - Tests.push_back(AllTests[tests[i]]); - } - - return true; + LogError() << "unknown test name: " << tests[i] << Endl; + return false; + } + Tests.push_back(AllTests[tests[i]]); + } + + return true; } - void Run() { + void Run() { #if !defined(NDEBUG) - LogInfo() << "*** DEBUG build! ***" << Endl; + LogInfo() << "*** DEBUG build! ***" << Endl; #endif - for (const TTest& test : Tests) { - LogInfo() << "Starting test " << test.Name << Endl; - - TInstant started = TInstant::Now(); - try { - test.Func(); - } catch (...) { - LogError() << "test " << test.Name - << " failed: " << CurrentExceptionMessage() - << Endl; - } - - LogInfo() << "List size = " << List.GetSize() << Endl; - - TDuration duration = TInstant::Now() - started; - LogInfo() << "test " << test.Name - << " duration: " << duration - << " (" << (double)duration.MicroSeconds() / (Iterations * NumWriters) << "us per iteration)" - << Endl; - LogInfo() << "Finished test " << test.Name << Endl; - } - } - - private: - void AddTest(const char* name, TTestFunc func) { - AllTests[name] = TTest(name, func); + for (const TTest& test : Tests) { + LogInfo() << "Starting test " << test.Name << Endl; + + TInstant started = TInstant::Now(); + try { + test.Func(); + } catch (...) { + LogError() << "test " << test.Name + << " failed: " << CurrentExceptionMessage() + << Endl; + } + + LogInfo() << "List size = " << List.GetSize() << Endl; + + TDuration duration = TInstant::Now() - started; + LogInfo() << "test " << test.Name + << " duration: " << duration + << " (" << (double)duration.MicroSeconds() / (Iterations * NumWriters) << "us per iteration)" + << Endl; + LogInfo() << "Finished test " << test.Name << Endl; + } } - TString PrintTests() const { - TVector<TString> names; - for (const auto& it : AllTests) { - names.push_back(it.first); - } - return JoinSeq(", ", names); + private: + void AddTest(const char* name, TTestFunc func) { + AllTests[name] = TTest(name, func); + } + + TString PrintTests() const { + TVector<TString> names; + for (const auto& it : AllTests) { + names.push_back(it.first); + } + return JoinSeq(", ", names); } - void TEST_Clear() { - List.Clear(); - } + void TEST_Clear() { + List.Clear(); + } - void TEST_InsertRandom() { - for (size_t i = 0; i < Iterations; ++i) { - List.Insert(TListItem(Random.GetString(KeyLen), Random.GetString(ValueLen))); - } + void TEST_InsertRandom() { + for (size_t i = 0; i < Iterations; ++i) { + List.Insert(TListItem(Random.GetString(KeyLen), Random.GetString(ValueLen))); + } } - void TEST_InsertSequential() { - TString key; - for (size_t i = 0; i < Iterations;) { - key.assign(Random.GetString(KeyLen)); - size_t batch = BatchSize / 2 + RandomNumber(BatchSize); - for (size_t j = 0; j < batch; ++j, ++i) { - key.resize(KeyLen - 1); - key.append((char)j); - List.Insert(TListItem(key, Random.GetString(ValueLen))); - } + void TEST_InsertSequential() { + TString key; + for (size_t i = 0; i < Iterations;) { + key.assign(Random.GetString(KeyLen)); + size_t batch = BatchSize / 2 + RandomNumber(BatchSize); + for (size_t j = 0; j < batch; ++j, ++i) { + key.resize(KeyLen - 1); + key.append((char)j); + List.Insert(TListItem(key, Random.GetString(ValueLen))); + } } } - void TEST_InsertSequentialSimple() { - for (size_t i = 0; i < Iterations; ++i) { - List.Insert(TListItem(Random.GetString(KeyLen), Random.GetString(ValueLen))); - } + void TEST_InsertSequentialSimple() { + for (size_t i = 0; i < Iterations; ++i) { + List.Insert(TListItem(Random.GetString(KeyLen), Random.GetString(ValueLen))); + } } - void TEST_LookupRandom() { - for (size_t i = 0; i < Iterations; ++i) { - List.SeekTo(TListItem(Random.GetString(KeyLen), TStringBuf())); - } + void TEST_LookupRandom() { + for (size_t i = 0; i < Iterations; ++i) { + List.SeekTo(TListItem(Random.GetString(KeyLen), TStringBuf())); + } } - void TEST_Concurrent() { - LogInfo() << "starting producers..." << Endl; - - TVector<TAutoPtr<TWorkerThread>> producers(NumWriters); - for (size_t i1 = 0; i1 < producers.size(); ++i1) { - producers[i1] = StartThread([&] { - TInstant started = TInstant::Now(); - for (size_t i2 = 0; i2 < Iterations; ++i2) { - { - TGuard<TMutex> guard(Mutex); - List.Insert(TListItem(Random.GetString(KeyLen), Random.GetString(ValueLen))); - } - } - TDuration duration = TInstant::Now() - started; - LogInfo() - << "Average time for producer = " - << (double)duration.MicroSeconds() / Iterations << "us per iteration" - << Endl; - }); - } - - LogInfo() << "starting consumers..." << Endl; - - TVector<TAutoPtr<TWorkerThread>> consumers(NumReaders); - for (size_t i1 = 0; i1 < consumers.size(); ++i1) { - consumers[i1] = StartThread([&] { - TInstant started = TInstant::Now(); - for (size_t i2 = 0; i2 < Iterations; ++i2) { - List.SeekTo(TListItem(Random.GetString(KeyLen), TStringBuf())); + void TEST_Concurrent() { + LogInfo() << "starting producers..." << Endl; + + TVector<TAutoPtr<TWorkerThread>> producers(NumWriters); + for (size_t i1 = 0; i1 < producers.size(); ++i1) { + producers[i1] = StartThread([&] { + TInstant started = TInstant::Now(); + for (size_t i2 = 0; i2 < Iterations; ++i2) { + { + TGuard<TMutex> guard(Mutex); + List.Insert(TListItem(Random.GetString(KeyLen), Random.GetString(ValueLen))); + } } - TDuration duration = TInstant::Now() - started; - LogInfo() - << "Average time for consumer = " - << (double)duration.MicroSeconds() / Iterations << "us per iteration" - << Endl; - }); - } - - LogInfo() << "wait for producers..." << Endl; - - TDuration producerTime; - for (size_t i = 0; i < producers.size(); ++i) { - producers[i]->Join(); - producerTime += producers[i]->GetTime(); - } - - LogInfo() << "wait for consumers..." << Endl; - - TDuration consumerTime; - for (size_t i = 0; i < consumers.size(); ++i) { - consumers[i]->Join(); - consumerTime += consumers[i]->GetTime(); - } - - LogInfo() << "average producer time: " - << producerTime.SecondsFloat() / producers.size() << " seconds" - << Endl; - - LogInfo() << "average consumer time: " - << consumerTime.SecondsFloat() / consumers.size() << " seconds" - << Endl; - } - }; - -} + TDuration duration = TInstant::Now() - started; + LogInfo() + << "Average time for producer = " + << (double)duration.MicroSeconds() / Iterations << "us per iteration" + << Endl; + }); + } + + LogInfo() << "starting consumers..." << Endl; + + TVector<TAutoPtr<TWorkerThread>> consumers(NumReaders); + for (size_t i1 = 0; i1 < consumers.size(); ++i1) { + consumers[i1] = StartThread([&] { + TInstant started = TInstant::Now(); + for (size_t i2 = 0; i2 < Iterations; ++i2) { + List.SeekTo(TListItem(Random.GetString(KeyLen), TStringBuf())); + } + TDuration duration = TInstant::Now() - started; + LogInfo() + << "Average time for consumer = " + << (double)duration.MicroSeconds() / Iterations << "us per iteration" + << Endl; + }); + } + + LogInfo() << "wait for producers..." << Endl; + + TDuration producerTime; + for (size_t i = 0; i < producers.size(); ++i) { + producers[i]->Join(); + producerTime += producers[i]->GetTime(); + } + + LogInfo() << "wait for consumers..." << Endl; + + TDuration consumerTime; + for (size_t i = 0; i < consumers.size(); ++i) { + consumers[i]->Join(); + consumerTime += consumers[i]->GetTime(); + } + + LogInfo() << "average producer time: " + << producerTime.SecondsFloat() / producers.size() << " seconds" + << Endl; + + LogInfo() << "average consumer time: " + << consumerTime.SecondsFloat() / consumers.size() << " seconds" + << Endl; + } + }; + +} //////////////////////////////////////////////////////////////////////////////// -int main(int argc, const char* argv[]) { +int main(int argc, const char* argv[]) { TTestSuite suite; if (!suite.Init(argc, argv)) { return -1; diff --git a/library/cpp/threading/skip_list/skiplist.h b/library/cpp/threading/skip_list/skiplist.h index 914a7c6ee7..c1ed46c4aa 100644 --- a/library/cpp/threading/skip_list/skiplist.h +++ b/library/cpp/threading/skip_list/skiplist.h @@ -10,399 +10,399 @@ #include <util/system/atomic.h> namespace NThreading { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - class TNopCounter { - protected: - template <typename T> - void OnInsert(const T&) { - } + class TNopCounter { + protected: + template <typename T> + void OnInsert(const T&) { + } - template <typename T> - void OnUpdate(const T&) { - } + template <typename T> + void OnUpdate(const T&) { + } - void Reset() { - } - }; + void Reset() { + } + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - class TSizeCounter { + class TSizeCounter { private: - size_t Size; + size_t Size; public: - TSizeCounter() - : Size(0) + TSizeCounter() + : Size(0) { } - size_t GetSize() const { - return Size; + size_t GetSize() const { + return Size; } - protected: - template <typename T> - void OnInsert(const T&) { - ++Size; + protected: + template <typename T> + void OnInsert(const T&) { + ++Size; } - template <typename T> - void OnUpdate(const T&) { + template <typename T> + void OnUpdate(const T&) { } - void Reset() { - Size = 0; + void Reset() { + Size = 0; } }; - //////////////////////////////////////////////////////////////////////////////// - // Append-only concurrent skip-list - // - // Readers do not require any synchronization. - // Writers should be externally synchronized. - // Nodes will be allocated using TMemoryPool instance. - - template < - typename T, - typename TComparer = TCompare<T>, - typename TAllocator = TMemoryPool, - typename TCounter = TSizeCounter, - int MaxHeight = 12, - int Branching = 4> - class TSkipList: public TCounter, private TNonCopyable { - class TNode { - private: - T Value; // should be immutable after insert - TNode* Next[]; // variable-size array maximum of MaxHeight values - - public: + //////////////////////////////////////////////////////////////////////////////// + // Append-only concurrent skip-list + // + // Readers do not require any synchronization. + // Writers should be externally synchronized. + // Nodes will be allocated using TMemoryPool instance. + + template < + typename T, + typename TComparer = TCompare<T>, + typename TAllocator = TMemoryPool, + typename TCounter = TSizeCounter, + int MaxHeight = 12, + int Branching = 4> + class TSkipList: public TCounter, private TNonCopyable { + class TNode { + private: + T Value; // should be immutable after insert + TNode* Next[]; // variable-size array maximum of MaxHeight values + + public: TNode(T&& value) : Value(std::move(value)) - { - Y_UNUSED(Next); - } - - const T& GetValue() const { - return Value; - } - - T& GetValue() { - return Value; - } - - TNode* GetNext(int height) const { - return AtomicGet(Next[height]); - } - - void Link(int height, TNode** prev) { - for (int i = 0; i < height; ++i) { - Next[i] = prev[i]->Next[i]; - AtomicSet(prev[i]->Next[i], this); - } - } - }; - + { + Y_UNUSED(Next); + } + + const T& GetValue() const { + return Value; + } + + T& GetValue() { + return Value; + } + + TNode* GetNext(int height) const { + return AtomicGet(Next[height]); + } + + void Link(int height, TNode** prev) { + for (int i = 0; i < height; ++i) { + Next[i] = prev[i]->Next[i]; + AtomicSet(prev[i]->Next[i], this); + } + } + }; + public: - class TIterator { - private: - const TSkipList* List; - const TNode* Node; - - public: - TIterator() - : List(nullptr) - , Node(nullptr) - { - } - - TIterator(const TSkipList* list, const TNode* node) - : List(list) - , Node(node) - { - } - - TIterator(const TIterator& other) - : List(other.List) - , Node(other.Node) - { + class TIterator { + private: + const TSkipList* List; + const TNode* Node; + + public: + TIterator() + : List(nullptr) + , Node(nullptr) + { + } + + TIterator(const TSkipList* list, const TNode* node) + : List(list) + , Node(node) + { + } + + TIterator(const TIterator& other) + : List(other.List) + , Node(other.Node) + { + } + + TIterator& operator=(const TIterator& other) { + List = other.List; + Node = other.Node; + return *this; + } + + void Next() { + Node = Node ? Node->GetNext(0) : nullptr; } - TIterator& operator=(const TIterator& other) { - List = other.List; - Node = other.Node; - return *this; - } + // much less efficient than Next as our list is single-linked + void Prev() { + if (Node) { + TNode* node = List->FindLessThan(Node->GetValue(), nullptr); + Node = (node != List->Head ? node : nullptr); + } + } - void Next() { - Node = Node ? Node->GetNext(0) : nullptr; - } + void Reset() { + Node = nullptr; + } - // much less efficient than Next as our list is single-linked - void Prev() { - if (Node) { - TNode* node = List->FindLessThan(Node->GetValue(), nullptr); - Node = (node != List->Head ? node : nullptr); - } - } - - void Reset() { - Node = nullptr; - } - - bool IsValid() const { - return Node != nullptr; - } + bool IsValid() const { + return Node != nullptr; + } - const T& GetValue() const { - Y_ASSERT(IsValid()); - return Node->GetValue(); - } - }; + const T& GetValue() const { + Y_ASSERT(IsValid()); + return Node->GetValue(); + } + }; - private: - TAllocator& Allocator; - TComparer Comparer; + private: + TAllocator& Allocator; + TComparer Comparer; - TNode* Head; - TAtomic Height; - TCounter Counter; + TNode* Head; + TAtomic Height; + TCounter Counter; - TNode* Prev[MaxHeight]; + TNode* Prev[MaxHeight]; - template <typename TValue> + template <typename TValue> using TComparerReturnType = std::invoke_result_t<TComparer, const T&, const TValue&>; - public: - TSkipList(TAllocator& allocator, const TComparer& comparer = TComparer()) - : Allocator(allocator) - , Comparer(comparer) - { - Init(); - } - - ~TSkipList() { - CallDtors(); - } - - void Clear() { - CallDtors(); - Allocator.ClearKeepFirstChunk(); - Init(); + public: + TSkipList(TAllocator& allocator, const TComparer& comparer = TComparer()) + : Allocator(allocator) + , Comparer(comparer) + { + Init(); + } + + ~TSkipList() { + CallDtors(); + } + + void Clear() { + CallDtors(); + Allocator.ClearKeepFirstChunk(); + Init(); } bool Insert(T value) { - TNode* node = PrepareInsert(value); - if (Y_UNLIKELY(node && Compare(node, value) == 0)) { - // we do not allow duplicates - return false; + TNode* node = PrepareInsert(value); + if (Y_UNLIKELY(node && Compare(node, value) == 0)) { + // we do not allow duplicates + return false; } node = DoInsert(std::move(value)); - TCounter::OnInsert(node->GetValue()); - return true; + TCounter::OnInsert(node->GetValue()); + return true; } - template <typename TInsertAction, typename TUpdateAction> - bool Insert(const T& value, TInsertAction insert, TUpdateAction update) { - TNode* node = PrepareInsert(value); - if (Y_UNLIKELY(node && Compare(node, value) == 0)) { - if (update(node->GetValue())) { - TCounter::OnUpdate(node->GetValue()); - return true; - } - // we do not allow duplicates - return false; - } - node = DoInsert(insert(value)); - TCounter::OnInsert(node->GetValue()); - return true; - } - - template <typename TValue> - bool Contains(const TValue& value) const { - TNode* node = FindGreaterThanOrEqual(value); - return node && Compare(node, value) == 0; - } - - TIterator SeekToFirst() const { - return TIterator(this, FindFirst()); - } - - TIterator SeekToLast() const { - TNode* last = FindLast(); - return TIterator(this, last != Head ? last : nullptr); - } - - template <typename TValue> - TIterator SeekTo(const TValue& value) const { - return TIterator(this, FindGreaterThanOrEqual(value)); + template <typename TInsertAction, typename TUpdateAction> + bool Insert(const T& value, TInsertAction insert, TUpdateAction update) { + TNode* node = PrepareInsert(value); + if (Y_UNLIKELY(node && Compare(node, value) == 0)) { + if (update(node->GetValue())) { + TCounter::OnUpdate(node->GetValue()); + return true; + } + // we do not allow duplicates + return false; + } + node = DoInsert(insert(value)); + TCounter::OnInsert(node->GetValue()); + return true; + } + + template <typename TValue> + bool Contains(const TValue& value) const { + TNode* node = FindGreaterThanOrEqual(value); + return node && Compare(node, value) == 0; + } + + TIterator SeekToFirst() const { + return TIterator(this, FindFirst()); + } + + TIterator SeekToLast() const { + TNode* last = FindLast(); + return TIterator(this, last != Head ? last : nullptr); + } + + template <typename TValue> + TIterator SeekTo(const TValue& value) const { + return TIterator(this, FindGreaterThanOrEqual(value)); } - private: - static int RandomHeight() { - int height = 1; - while (height < MaxHeight && (RandomNumber<unsigned int>() % Branching) == 0) { - ++height; - } - return height; + private: + static int RandomHeight() { + int height = 1; + while (height < MaxHeight && (RandomNumber<unsigned int>() % Branching) == 0) { + ++height; + } + return height; + } + + void Init() { + Head = AllocateRootNode(); + Height = 1; + TCounter::Reset(); + + for (int i = 0; i < MaxHeight; ++i) { + Prev[i] = Head; + } } - void Init() { - Head = AllocateRootNode(); - Height = 1; - TCounter::Reset(); - - for (int i = 0; i < MaxHeight; ++i) { - Prev[i] = Head; + void CallDtors() { + if (!TTypeTraits<T>::IsPod) { + // we should explicitly call destructors for our nodes + TNode* node = Head->GetNext(0); + while (node) { + TNode* next = node->GetNext(0); + node->~TNode(); + node = next; + } } } - void CallDtors() { - if (!TTypeTraits<T>::IsPod) { - // we should explicitly call destructors for our nodes - TNode* node = Head->GetNext(0); - while (node) { - TNode* next = node->GetNext(0); - node->~TNode(); - node = next; - } - } - } - - TNode* AllocateRootNode() { - size_t size = sizeof(TNode) + sizeof(TNode*) * MaxHeight; - void* buffer = Allocator.Allocate(size); - memset(buffer, 0, size); - return static_cast<TNode*>(buffer); - } + TNode* AllocateRootNode() { + size_t size = sizeof(TNode) + sizeof(TNode*) * MaxHeight; + void* buffer = Allocator.Allocate(size); + memset(buffer, 0, size); + return static_cast<TNode*>(buffer); + } TNode* AllocateNode(T&& value, int height) { - size_t size = sizeof(TNode) + sizeof(TNode*) * height; - void* buffer = Allocator.Allocate(size); - memset(buffer, 0, size); + size_t size = sizeof(TNode) + sizeof(TNode*) * height; + void* buffer = Allocator.Allocate(size); + memset(buffer, 0, size); return new (buffer) TNode(std::move(value)); - } - - TNode* FindFirst() const { - return Head->GetNext(0); - } - - TNode* FindLast() const { - TNode* node = Head; - int height = AtomicGet(Height) - 1; - - while (true) { - TNode* next = node->GetNext(height); - if (next) { - node = next; - continue; - } - - if (height) { - --height; - } else { - return node; - } + } + + TNode* FindFirst() const { + return Head->GetNext(0); + } + + TNode* FindLast() const { + TNode* node = Head; + int height = AtomicGet(Height) - 1; + + while (true) { + TNode* next = node->GetNext(height); + if (next) { + node = next; + continue; + } + + if (height) { + --height; + } else { + return node; + } } } - template <typename TValue> - TComparerReturnType<TValue> Compare(const TNode* node, const TValue& value) const { - return Comparer(node->GetValue(), value); - } - - template <typename TValue> - TNode* FindLessThan(const TValue& value, TNode** links) const { - TNode* node = Head; - int height = AtomicGet(Height) - 1; - - TNode* prev = nullptr; - while (true) { - TNode* next = node->GetNext(height); - if (next && next != prev) { - TComparerReturnType<TValue> cmp = Compare(next, value); - if (cmp < 0) { - node = next; - continue; - } + template <typename TValue> + TComparerReturnType<TValue> Compare(const TNode* node, const TValue& value) const { + return Comparer(node->GetValue(), value); + } + + template <typename TValue> + TNode* FindLessThan(const TValue& value, TNode** links) const { + TNode* node = Head; + int height = AtomicGet(Height) - 1; + + TNode* prev = nullptr; + while (true) { + TNode* next = node->GetNext(height); + if (next && next != prev) { + TComparerReturnType<TValue> cmp = Compare(next, value); + if (cmp < 0) { + node = next; + continue; + } } - if (links) { - // collect links from upper levels - links[height] = node; - } - - if (height) { - prev = next; - --height; - } else { - return node; - } + if (links) { + // collect links from upper levels + links[height] = node; + } + + if (height) { + prev = next; + --height; + } else { + return node; + } } } - template <typename TValue> - TNode* FindGreaterThanOrEqual(const TValue& value) const { - TNode* node = Head; - int height = AtomicGet(Height) - 1; - - TNode* prev = nullptr; - while (true) { - TNode* next = node->GetNext(height); - if (next && next != prev) { - TComparerReturnType<TValue> cmp = Compare(next, value); - if (cmp < 0) { - node = next; - continue; - } - if (cmp == 0) { - return next; - } + template <typename TValue> + TNode* FindGreaterThanOrEqual(const TValue& value) const { + TNode* node = Head; + int height = AtomicGet(Height) - 1; + + TNode* prev = nullptr; + while (true) { + TNode* next = node->GetNext(height); + if (next && next != prev) { + TComparerReturnType<TValue> cmp = Compare(next, value); + if (cmp < 0) { + node = next; + continue; + } + if (cmp == 0) { + return next; + } } - - if (height) { - prev = next; - --height; - } else { + + if (height) { + prev = next; + --height; + } else { return next; } } - } + } - TNode* PrepareInsert(const T& value) { - TNode* prev = Prev[0]; - TNode* next = prev->GetNext(0); - if ((prev == Head || Compare(prev, value) < 0) && (next == nullptr || Compare(next, value) >= 0)) { - // avoid seek in case of sequential insert + TNode* PrepareInsert(const T& value) { + TNode* prev = Prev[0]; + TNode* next = prev->GetNext(0); + if ((prev == Head || Compare(prev, value) < 0) && (next == nullptr || Compare(next, value) >= 0)) { + // avoid seek in case of sequential insert } else { - prev = FindLessThan(value, Prev); - next = prev->GetNext(0); + prev = FindLessThan(value, Prev); + next = prev->GetNext(0); } - return next; + return next; } TNode* DoInsert(T&& value) { - // choose level to place new node - int currentHeight = AtomicGet(Height); - int height = RandomHeight(); - if (height > currentHeight) { - for (int i = currentHeight; i < height; ++i) { - // head should link to all levels - Prev[i] = Head; - } - AtomicSet(Height, height); + // choose level to place new node + int currentHeight = AtomicGet(Height); + int height = RandomHeight(); + if (height > currentHeight) { + for (int i = currentHeight; i < height; ++i) { + // head should link to all levels + Prev[i] = Head; + } + AtomicSet(Height, height); } TNode* node = AllocateNode(std::move(value), height); - node->Link(height, Prev); + node->Link(height, Prev); - // keep last inserted node to optimize sequential inserts - for (int i = 0; i < height; i++) { - Prev[i] = node; - } - return node; + // keep last inserted node to optimize sequential inserts + for (int i = 0; i < height; i++) { + Prev[i] = node; + } + return node; } - }; + }; -} +} diff --git a/library/cpp/threading/skip_list/skiplist_ut.cpp b/library/cpp/threading/skip_list/skiplist_ut.cpp index 52fcffda66..fdc831dffd 100644 --- a/library/cpp/threading/skip_list/skiplist_ut.cpp +++ b/library/cpp/threading/skip_list/skiplist_ut.cpp @@ -3,41 +3,41 @@ #include <library/cpp/testing/unittest/registar.h> namespace NThreading { - namespace { - struct TTestObject { - static size_t Count; - int Tag; - - TTestObject(int tag) - : Tag(tag) - { - ++Count; - } - - TTestObject(const TTestObject& other) - : Tag(other.Tag) - { - ++Count; - } - - ~TTestObject() { - --Count; - } - - bool operator<(const TTestObject& other) const { - return Tag < other.Tag; - } - }; - - size_t TTestObject::Count = 0; + namespace { + struct TTestObject { + static size_t Count; + int Tag; + + TTestObject(int tag) + : Tag(tag) + { + ++Count; + } + + TTestObject(const TTestObject& other) + : Tag(other.Tag) + { + ++Count; + } + + ~TTestObject() { + --Count; + } + + bool operator<(const TTestObject& other) const { + return Tag < other.Tag; + } + }; + + size_t TTestObject::Count = 0; } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// Y_UNIT_TEST_SUITE(TSkipListTest) { Y_UNIT_TEST(ShouldBeEmptyAfterCreation) { - TMemoryPool pool(1024); + TMemoryPool pool(1024); TSkipList<int> list(pool); UNIT_ASSERT_EQUAL(list.GetSize(), 0); @@ -182,4 +182,4 @@ namespace NThreading { } } -} +} diff --git a/library/cpp/threading/task_scheduler/task_scheduler_ut.cpp b/library/cpp/threading/task_scheduler/task_scheduler_ut.cpp index 3b5203194a..2223dce650 100644 --- a/library/cpp/threading/task_scheduler/task_scheduler_ut.cpp +++ b/library/cpp/threading/task_scheduler/task_scheduler_ut.cpp @@ -14,9 +14,9 @@ class TTaskSchedulerTest: public TTestBase { class TCheckTask: public TTaskScheduler::IRepeatedTask { public: - TCheckTask(const TDuration& delay) - : Start_(Now()) - , Delay_(delay) + TCheckTask(const TDuration& delay) + : Start_(Now()) + , Delay_(delay) { AtomicIncrement(ScheduledTaskCounter_); } @@ -25,15 +25,15 @@ class TTaskSchedulerTest: public TTestBase { } bool Process() override { - const TDuration delay = Now() - Start_; + const TDuration delay = Now() - Start_; - if (delay < Delay_) { + if (delay < Delay_) { AtomicIncrement(BadTimeoutCounter_); } AtomicIncrement(ExecutedTaskCounter_); - - return false; + + return false; } static bool AllTaskExecuted() { @@ -45,8 +45,8 @@ class TTaskSchedulerTest: public TTestBase { } private: - TInstant Start_; - TDuration Delay_; + TInstant Start_; + TDuration Delay_; static TAtomic BadTimeoutCounter_; static TAtomic ScheduledTaskCounter_; static TAtomic ExecutedTaskCounter_; @@ -60,7 +60,7 @@ class TTaskSchedulerTest: public TTestBase { ScheduleCheckTask(10000); ScheduleCheckTask(5000); - Scheduler_.Start(); + Scheduler_.Start(); usleep(1000000); @@ -70,8 +70,8 @@ class TTaskSchedulerTest: public TTestBase { private: void ScheduleCheckTask(size_t delay) { - TDuration d = TDuration::MicroSeconds(delay); - + TDuration d = TDuration::MicroSeconds(delay); + Scheduler_.Add(new TCheckTask(d), d); } diff --git a/library/cpp/timezone_conversion/civil-inl.h b/library/cpp/timezone_conversion/civil-inl.h index 32afbccb64..03c9de4ba0 100644 --- a/library/cpp/timezone_conversion/civil-inl.h +++ b/library/cpp/timezone_conversion/civil-inl.h @@ -7,27 +7,27 @@ namespace NDatetime { template <typename T> struct TGetCivilUnit; - template <> + template <> struct TGetCivilUnit<TCivilSecond> { static constexpr ECivilUnit Value = ECivilUnit::Second; }; - template <> + template <> struct TGetCivilUnit<TCivilMinute> { static constexpr ECivilUnit Value = ECivilUnit::Minute; }; - template <> + template <> struct TGetCivilUnit<TCivilHour> { static constexpr ECivilUnit Value = ECivilUnit::Hour; }; - template <> + template <> struct TGetCivilUnit<TCivilDay> { static constexpr ECivilUnit Value = ECivilUnit::Day; }; - template <> + template <> struct TGetCivilUnit<TCivilMonth> { static constexpr ECivilUnit Value = ECivilUnit::Month; }; - template <> + template <> struct TGetCivilUnit<TCivilYear> { static constexpr ECivilUnit Value = ECivilUnit::Year; }; @@ -35,29 +35,29 @@ namespace NDatetime { template <ECivilUnit Unit> struct TGetCivilTime; - template <> + template <> struct TGetCivilTime<ECivilUnit::Second> { using TResult = TCivilSecond; }; - template <> + template <> struct TGetCivilTime<ECivilUnit::Minute> { using TResult = TCivilMinute; }; - template <> + template <> struct TGetCivilTime<ECivilUnit::Hour> { using TResult = TCivilHour; }; - template <> + template <> struct TGetCivilTime<ECivilUnit::Day> { using TResult = TCivilDay; }; - template <> + template <> struct TGetCivilTime<ECivilUnit::Month> { using TResult = TCivilMonth; }; - template <> + template <> struct TGetCivilTime<ECivilUnit::Year> { using TResult = TCivilYear; }; - } -} + } +} diff --git a/library/cpp/timezone_conversion/civil.cpp b/library/cpp/timezone_conversion/civil.cpp index 5986318b9a..3cecf0b467 100644 --- a/library/cpp/timezone_conversion/civil.cpp +++ b/library/cpp/timezone_conversion/civil.cpp @@ -177,7 +177,7 @@ namespace NDatetime { template <> void Out<NDatetime::TCivilYear>(IOutputStream& out, const NDatetime::TCivilYear& y) { - out << y.year(); + out << y.year(); } template <> @@ -209,26 +209,26 @@ template <> void Out<NDatetime::TWeekday>(IOutputStream& out, NDatetime::TWeekday wd) { using namespace cctz; switch (wd) { - case weekday::monday: + case weekday::monday: out << TStringBuf("Monday"); - break; - case weekday::tuesday: + break; + case weekday::tuesday: out << TStringBuf("Tuesday"); - break; - case weekday::wednesday: + break; + case weekday::wednesday: out << TStringBuf("Wednesday"); - break; - case weekday::thursday: + break; + case weekday::thursday: out << TStringBuf("Thursday"); - break; - case weekday::friday: + break; + case weekday::friday: out << TStringBuf("Friday"); - break; - case weekday::saturday: + break; + case weekday::saturday: out << TStringBuf("Saturday"); - break; - case weekday::sunday: + break; + case weekday::sunday: out << TStringBuf("Sunday"); - break; + break; } } diff --git a/library/cpp/timezone_conversion/civil.h b/library/cpp/timezone_conversion/civil.h index 0e95b807ed..edcc428a78 100644 --- a/library/cpp/timezone_conversion/civil.h +++ b/library/cpp/timezone_conversion/civil.h @@ -58,7 +58,7 @@ namespace NDatetime { template <ECivilUnit Unit> struct TGetCivilTime; - } + } template <typename T> CONSTEXPR_M ECivilUnit GetCivilUnit(const T& = {}) { diff --git a/library/cpp/timezone_conversion/convert.cpp b/library/cpp/timezone_conversion/convert.cpp index 490bb4e270..e3b789a3dd 100644 --- a/library/cpp/timezone_conversion/convert.cpp +++ b/library/cpp/timezone_conversion/convert.cpp @@ -26,27 +26,27 @@ namespace NDatetime { tm.IsLeap = LeapYearAD(cs.year()); switch (cctz::get_weekday(cd)) { - case cctz::weekday::monday: - tm.WDay = 1; - break; - case cctz::weekday::tuesday: - tm.WDay = 2; - break; - case cctz::weekday::wednesday: - tm.WDay = 3; - break; - case cctz::weekday::thursday: - tm.WDay = 4; - break; - case cctz::weekday::friday: - tm.WDay = 5; - break; - case cctz::weekday::saturday: - tm.WDay = 6; - break; - case cctz::weekday::sunday: - tm.WDay = 0; - break; + case cctz::weekday::monday: + tm.WDay = 1; + break; + case cctz::weekday::tuesday: + tm.WDay = 2; + break; + case cctz::weekday::wednesday: + tm.WDay = 3; + break; + case cctz::weekday::thursday: + tm.WDay = 4; + break; + case cctz::weekday::friday: + tm.WDay = 5; + break; + case cctz::weekday::saturday: + tm.WDay = 6; + break; + case cctz::weekday::sunday: + tm.WDay = 0; + break; } return tm; @@ -87,7 +87,7 @@ namespace NDatetime { civilTime.MDay, civilTime.Hour, civilTime.Min, - civilTime.Sec); + civilTime.Sec); return TInstant::Seconds(TSystemClock::to_time_t(tz.lookup(cs).pre)); } } diff --git a/library/cpp/timezone_conversion/ut/civil_ut.cpp b/library/cpp/timezone_conversion/ut/civil_ut.cpp index a21bd4bd7d..b2e29b6943 100644 --- a/library/cpp/timezone_conversion/ut/civil_ut.cpp +++ b/library/cpp/timezone_conversion/ut/civil_ut.cpp @@ -5,10 +5,10 @@ #include <util/stream/str.h> namespace NDatetime { - inline bool operator==(const NDatetime::TCivilDiff& x, const NDatetime::TCivilDiff& y) { + inline bool operator==(const NDatetime::TCivilDiff& x, const NDatetime::TCivilDiff& y) { return x.Unit == y.Unit && x.Value == y.Value; } -} +} template <> inline void Out<NDatetime::TCivilDiff>(IOutputStream& out, const NDatetime::TCivilDiff& diff) { diff --git a/library/cpp/timezone_conversion/ut/convert_ut.cpp b/library/cpp/timezone_conversion/ut/convert_ut.cpp index bbf9e9b826..477813e542 100644 --- a/library/cpp/timezone_conversion/ut/convert_ut.cpp +++ b/library/cpp/timezone_conversion/ut/convert_ut.cpp @@ -29,11 +29,11 @@ void CompareCivilTimes(const TSimpleTM& expected, const TSimpleTM& actual) { } #define CHECK_ROUND_TRIP(tz, unixTime, civil) \ - EXPECT_EQ( \ - TInstant::Seconds(unixTime), \ - ToAbsoluteTime(civil, tz)); \ + EXPECT_EQ( \ + TInstant::Seconds(unixTime), \ + ToAbsoluteTime(civil, tz)); \ CompareCivilTimes( \ - civil, \ + civil, \ ToCivilTime(TInstant::Seconds(unixTime), tz)); // Tests only unambiguous civil times (i.e., those that occurred exactly once). @@ -41,35 +41,35 @@ TEST(TimeZoneConversion, Simple) { TTimeZone msk = GetTimeZone("Europe/Moscow"); // Before and after the temporary switch to UTC+3 in 2010. CHECK_ROUND_TRIP( - msk, - 1288475999, - ZonedTm(+4, true, 2010, 10, 31, 1, 59, 59)); + msk, + 1288475999, + ZonedTm(+4, true, 2010, 10, 31, 1, 59, 59)); CHECK_ROUND_TRIP( - msk, - 1288475999 + 3 * 60 * 60, - ZonedTm(+3, false, 2010, 10, 31, 3, 59, 59)); + msk, + 1288475999 + 3 * 60 * 60, + ZonedTm(+3, false, 2010, 10, 31, 3, 59, 59)); // Before and after the permanent switch to UTC+4 in 2011. CHECK_ROUND_TRIP( - msk, - 1301180399, - ZonedTm(+3, false, 2011, 3, 27, 1, 59, 59)); + msk, + 1301180399, + ZonedTm(+3, false, 2011, 3, 27, 1, 59, 59)); CHECK_ROUND_TRIP( - msk, - 1301180399 + 60 * 60, - ZonedTm(+4, false, 2011, 3, 27, 3, 59, 59)); + msk, + 1301180399 + 60 * 60, + ZonedTm(+4, false, 2011, 3, 27, 3, 59, 59)); // Some random moment between 2011 and 2014 when UTC+4 (no DST) was in place. CHECK_ROUND_TRIP( - msk, - 1378901234, - ZonedTm(+4, false, 2013, 9, 11, 16, 7, 14)); + msk, + 1378901234, + ZonedTm(+4, false, 2013, 9, 11, 16, 7, 14)); // As of right now (i.e., as I'm writing this) Moscow is in UTC+3 (no DST). CHECK_ROUND_TRIP( - msk, - 1458513396, - ZonedTm(+3, false, 2016, 3, 21, 1, 36, 36)); + msk, + 1458513396, + ZonedTm(+3, false, 2016, 3, 21, 1, 36, 36)); // Please add a new test if the current president moves Moscow back to UTC+4 // or introduces DST again. @@ -122,9 +122,9 @@ TEST(TimeZoneConversion, TestSkippedDate) { TEST(TimeZoneConversion, Utc) { CHECK_ROUND_TRIP( - GetUtcTimeZone(), - 1451703845, - ZonedTm(0, false, 2016, 1, 2, 3, 4, 5)); + GetUtcTimeZone(), + 1451703845, + ZonedTm(0, false, 2016, 1, 2, 3, 4, 5)); } TEST(TimeZoneConversion, Local) { @@ -182,7 +182,7 @@ TEST(TimeZoneConversion, TestFutureDstChanges) { // 31 days later we're back to UTC+0 again. CHECK_ROUND_TRIP( london, - 1760124660 + 31 * 24 * 60 * 60, + 1760124660 + 31 * 24 * 60 * 60, ZonedTm(+0, false, 2025, 11, 10, 19, 31, 0)); } diff --git a/library/cpp/tld/gen_tld.py b/library/cpp/tld/gen_tld.py index 882b701e1d..59b265fb21 100755 --- a/library/cpp/tld/gen_tld.py +++ b/library/cpp/tld/gen_tld.py @@ -27,7 +27,7 @@ def main(): break print '// actual list can be found at http://data.iana.org/TLD/tlds-alpha-by-domain.txt' - print 'static const char* const TopLevelDomains[] = {' + print 'static const char* const TopLevelDomains[] = {' for label, value in sorted(tlds.iteritems()): if label == 'xn--': diff --git a/library/cpp/tld/tld.cpp b/library/cpp/tld/tld.cpp index e31f3f0322..dbfd9d5332 100644 --- a/library/cpp/tld/tld.cpp +++ b/library/cpp/tld/tld.cpp @@ -1,48 +1,48 @@ #include "tld.h" #include <library/cpp/digest/lower_case/hash_ops.h> - + #include <util/generic/hash_set.h> -#include <util/generic/singleton.h> +#include <util/generic/singleton.h> namespace NTld { - namespace { + namespace { #include <library/cpp/tld/tld.inc> - using TCiHash = THashSet<TStringBuf, TCIOps, TCIOps>; - - struct TTLDHash: public TCiHash { - TTLDHash() { - for (auto tld = GetTlds(); *tld; ++tld) { - insert(*tld); - } - } - }; - - struct TVeryGoodTld: public TCiHash { - TVeryGoodTld() { - auto domains = { - "am", "az", "biz", "by", "com", "cz", "de", "ec", "fr", "ge", "gov", - "gr", "il", "info", "kg", "kz", "mobi", "net", "nu", "org", "lt", "lv", - "md", "ru", "su", "tr", "ua", "uk", "uz", "ws", "xn--p1ai", "рф"}; - - for (auto d : domains) { - insert(d); - } + using TCiHash = THashSet<TStringBuf, TCIOps, TCIOps>; + + struct TTLDHash: public TCiHash { + TTLDHash() { + for (auto tld = GetTlds(); *tld; ++tld) { + insert(*tld); + } + } + }; + + struct TVeryGoodTld: public TCiHash { + TVeryGoodTld() { + auto domains = { + "am", "az", "biz", "by", "com", "cz", "de", "ec", "fr", "ge", "gov", + "gr", "il", "info", "kg", "kz", "mobi", "net", "nu", "org", "lt", "lv", + "md", "ru", "su", "tr", "ua", "uk", "uz", "ws", "xn--p1ai", "рф"}; + + for (auto d : domains) { + insert(d); + } } - }; - } - - const char* const* GetTlds() { - return TopLevelDomains; - } - - bool IsTld(const TStringBuf& s) { + }; + } + + const char* const* GetTlds() { + return TopLevelDomains; + } + + bool IsTld(const TStringBuf& s) { return Default<TTLDHash>().contains(s); - } + } - bool IsVeryGoodTld(const TStringBuf& s) { + bool IsVeryGoodTld(const TStringBuf& s) { return Default<TVeryGoodTld>().contains(s); - } - + } + } diff --git a/library/cpp/tld/tld.h b/library/cpp/tld/tld.h index 9e241de090..4829b7bea6 100644 --- a/library/cpp/tld/tld.h +++ b/library/cpp/tld/tld.h @@ -3,26 +3,26 @@ #include <util/generic/strbuf.h> namespace NTld { - const char* const* GetTlds(); + const char* const* GetTlds(); - // Note that FindTld() returns empty string when @host is single domain label (without '.'). + // Note that FindTld() returns empty string when @host is single domain label (without '.'). // If you need whole @host for such case, you can use GetZone() from library/cpp/string_utils/url/url.h - inline TStringBuf FindTld(const TStringBuf& host) { - size_t p = host.rfind('.'); - return p != TStringBuf::npos ? host.SubStr(p + 1) : TStringBuf(); - } + inline TStringBuf FindTld(const TStringBuf& host) { + size_t p = host.rfind('.'); + return p != TStringBuf::npos ? host.SubStr(p + 1) : TStringBuf(); + } - bool IsTld(const TStringBuf& tld); + bool IsTld(const TStringBuf& tld); - inline bool InTld(const TStringBuf& host) { - return IsTld(FindTld(host)); - } + inline bool InTld(const TStringBuf& host) { + return IsTld(FindTld(host)); + } - // check if @s belongs to a "good" subset of reliable TLDs, defined in tld.cpp - bool IsVeryGoodTld(const TStringBuf& tld); + // check if @s belongs to a "good" subset of reliable TLDs, defined in tld.cpp + bool IsVeryGoodTld(const TStringBuf& tld); - inline bool InVeryGoodTld(const TStringBuf& host) { - return IsVeryGoodTld(FindTld(host)); - } + inline bool InVeryGoodTld(const TStringBuf& host) { + return IsVeryGoodTld(FindTld(host)); + } } diff --git a/library/cpp/tld/tld_ut.cpp b/library/cpp/tld/tld_ut.cpp index 733200f2b5..9d11e3adaa 100644 --- a/library/cpp/tld/tld_ut.cpp +++ b/library/cpp/tld/tld_ut.cpp @@ -43,9 +43,9 @@ Y_UNIT_TEST_SUITE(TTldTest) { UNIT_ASSERT(!IsVeryGoodTld("BMW")); UNIT_ASSERT(!IsVeryGoodTld("TiReS")); UNIT_ASSERT(IsVeryGoodTld("рф")); - UNIT_ASSERT(!IsVeryGoodTld("РФ")); // note that uppercase non-ascii tlds cannot be found - UNIT_ASSERT(IsVeryGoodTld("xn--p1ai")); // "рф" - UNIT_ASSERT(!IsVeryGoodTld("xn--p1ag")); // "ру" + UNIT_ASSERT(!IsVeryGoodTld("РФ")); // note that uppercase non-ascii tlds cannot be found + UNIT_ASSERT(IsVeryGoodTld("xn--p1ai")); // "рф" + UNIT_ASSERT(!IsVeryGoodTld("xn--p1ag")); // "ру" UNIT_ASSERT(!IsVeryGoodTld("YaHOO")); UNIT_ASSERT(!IsVeryGoodTld("xn")); diff --git a/library/cpp/tld/ya.make b/library/cpp/tld/ya.make index aeabbfeebf..1ff1001a17 100644 --- a/library/cpp/tld/ya.make +++ b/library/cpp/tld/ya.make @@ -7,7 +7,7 @@ PYTHON( IN tlds-alpha-by-domain.txt STDOUT tld.inc ) - + SRCS( tld.cpp ) diff --git a/library/cpp/unicode/normalization/custom_encoder.cpp b/library/cpp/unicode/normalization/custom_encoder.cpp index c6f186405f..897667861c 100644 --- a/library/cpp/unicode/normalization/custom_encoder.cpp +++ b/library/cpp/unicode/normalization/custom_encoder.cpp @@ -9,7 +9,7 @@ void TCustomEncoder::addToTable(wchar32 ucode, unsigned char code, const CodePag unsigned char pos = (unsigned char)(ucode & 255); if (Table[plane] == DefaultPlane) { Table[plane] = new char[256]; - memset(Table[plane], 0, 256 * sizeof(char)); + memset(Table[plane], 0, 256 * sizeof(char)); } if (Table[plane][pos] == 0) { @@ -18,18 +18,18 @@ void TCustomEncoder::addToTable(wchar32 ucode, unsigned char code, const CodePag Y_ASSERT(target && *target->Names); if (static_cast<unsigned char>(Table[plane][pos]) > 127 && code) { Cerr << "WARNING: Only lower part of ASCII should have duplicate encodings " - << target->Names[0] - << " " << IntToString<16>(ucode) - << " " << IntToString<16>(code) - << " " << IntToString<16>(static_cast<unsigned char>(Table[plane][pos])) - << Endl; + << target->Names[0] + << " " << IntToString<16>(ucode) + << " " << IntToString<16>(code) + << " " << IntToString<16>(static_cast<unsigned char>(Table[plane][pos])) + << Endl; } } } -bool isGoodDecomp(wchar32 rune, wchar32 decomp) { +bool isGoodDecomp(wchar32 rune, wchar32 decomp) { if ( - (NUnicode::NPrivate::CharInfo(rune) == NUnicode::NPrivate::CharInfo(decomp)) || (IsAlpha(rune) && IsAlpha(decomp)) || (IsNumeric(rune) && IsNumeric(decomp)) || (IsQuotation(rune) && IsQuotation(decomp))) + (NUnicode::NPrivate::CharInfo(rune) == NUnicode::NPrivate::CharInfo(decomp)) || (IsAlpha(rune) && IsAlpha(decomp)) || (IsNumeric(rune) && IsNumeric(decomp)) || (IsQuotation(rune) && IsQuotation(decomp))) { return true; } @@ -43,23 +43,23 @@ void TCustomEncoder::Create(const CodePage* target, bool extended) { DefaultPlane = new char[256]; - memset(DefaultPlane, 0, 256 * sizeof(char)); + memset(DefaultPlane, 0, 256 * sizeof(char)); for (size_t i = 0; i != 256; ++i) Table[i] = DefaultPlane; for (size_t i = 0; i != 256; ++i) { wchar32 ucode = target->unicode[i]; if (ucode != BROKEN_RUNE) // always UNASSIGNED - addToTable(ucode, (unsigned char)i, target); + addToTable(ucode, (unsigned char)i, target); } if (!extended) return; - for (wchar32 w = 1; w < 65535; w++) { - if (Code(w) == 0) { + for (wchar32 w = 1; w < 65535; w++) { + if (Code(w) == 0) { wchar32 dw = w; - while (IsComposed(dw) && Code(dw) == 0) { + while (IsComposed(dw) && Code(dw) == 0) { const wchar32* decomp_p = NUnicode::Decomposition<true>(dw); Y_ASSERT(decomp_p != nullptr); @@ -76,8 +76,8 @@ void TCustomEncoder::Create(const CodePage* target, bool extended) { TCustomEncoder::~TCustomEncoder() { for (size_t i = 0; i != 256; ++i) { if (Table[i] != DefaultPlane) { - delete[] Table[i]; + delete[] Table[i]; } } - delete[] DefaultPlane; + delete[] DefaultPlane; } diff --git a/library/cpp/unicode/normalization/custom_encoder.h b/library/cpp/unicode/normalization/custom_encoder.h index ef4d5b7f65..28644f37fa 100644 --- a/library/cpp/unicode/normalization/custom_encoder.h +++ b/library/cpp/unicode/normalization/custom_encoder.h @@ -2,8 +2,8 @@ #include <library/cpp/charset/codepage.h> -struct TCustomEncoder: public Encoder { - void Create(const CodePage* target, bool extended = false); +struct TCustomEncoder: public Encoder { + void Create(const CodePage* target, bool extended = false); ~TCustomEncoder(); private: diff --git a/library/cpp/unicode/normalization/decomposition_table.h b/library/cpp/unicode/normalization/decomposition_table.h index 23f3da334f..5a0b30d078 100644 --- a/library/cpp/unicode/normalization/decomposition_table.h +++ b/library/cpp/unicode/normalization/decomposition_table.h @@ -3,26 +3,26 @@ #include <util/charset/unicode_table.h> namespace NUnicode { - namespace NPrivate { - typedef NUnicodeTable::TTable<NUnicodeTable::TSubtable< - NUnicodeTable::UNICODE_TABLE_SHIFT, NUnicodeTable::TValues<const wchar32*>>> - TDecompositionTable; + namespace NPrivate { + typedef NUnicodeTable::TTable<NUnicodeTable::TSubtable< + NUnicodeTable::UNICODE_TABLE_SHIFT, NUnicodeTable::TValues<const wchar32*>>> + TDecompositionTable; - const TDecompositionTable& CannonDecompositionTable(); - const TDecompositionTable& CompatDecompositionTable(); + const TDecompositionTable& CannonDecompositionTable(); + const TDecompositionTable& CompatDecompositionTable(); - template <bool compat> - inline const TDecompositionTable& DecompositionTable(); + template <bool compat> + inline const TDecompositionTable& DecompositionTable(); - template <> - inline const TDecompositionTable& DecompositionTable<false>() { - return CannonDecompositionTable(); - } + template <> + inline const TDecompositionTable& DecompositionTable<false>() { + return CannonDecompositionTable(); + } - template <> - inline const TDecompositionTable& DecompositionTable<true>() { - return CompatDecompositionTable(); - } + template <> + inline const TDecompositionTable& DecompositionTable<true>() { + return CompatDecompositionTable(); + } - } -}; // namespace NUnicode + } +}; // namespace NUnicode diff --git a/library/cpp/unicode/normalization/normalization.h b/library/cpp/unicode/normalization/normalization.h index 4f5f57881c..9fd813b761 100644 --- a/library/cpp/unicode/normalization/normalization.h +++ b/library/cpp/unicode/normalization/normalization.h @@ -40,20 +40,20 @@ namespace NUnicode { public: inline TDecompositor(const TDecompositionTable& table) : Table(table) - { - } + { + } inline const wchar32* Decomposition(wchar32 ch) const { return NPrivate::Decomposition(Table, ch); } }; - template <bool IsCompat> + template <bool IsCompat> struct TStandartDecompositor: public TDecompositor { TStandartDecompositor() - : TDecompositor(NPrivate::DecompositionTable<IsCompat>()) - { - } + : TDecompositor(NPrivate::DecompositionTable<IsCompat>()) + { + } }; template <ENormalization N> @@ -92,12 +92,12 @@ namespace NUnicode { static const TRawData RawData[]; static const size_t RawDataSize; - class TKey: public std::pair<wchar32, wchar32> { + class TKey: public std::pair<wchar32, wchar32> { public: inline TKey(wchar32 a, wchar32 b) : std::pair<wchar32, wchar32>(a, b) - { - } + { + } inline size_t Hash() const { return CombineHashes(first, second); @@ -111,7 +111,7 @@ namespace NUnicode { } }; - typedef THashMap<TKey, wchar32, THash<TKey>> TData; + typedef THashMap<TKey, wchar32, THash<TKey>> TData; TData Data; public: @@ -149,8 +149,8 @@ namespace NUnicode { public: inline TCompositor() : Composition(Singleton<TComposition>()) - { - } + { + } inline void DoComposition(TBuffer& buffer) { if (buffer.size() < 2) @@ -204,7 +204,7 @@ namespace NUnicode { } return true; } - } + } template <bool compat> inline const wchar32* Decomposition(wchar32 ch) { @@ -232,7 +232,7 @@ namespace NUnicode { } struct TComparer { - inline bool operator()(const TSymbol& a, const TSymbol& b) { + inline bool operator()(const TSymbol& a, const TSymbol& b) { return Compare(a, b); } }; @@ -304,14 +304,14 @@ namespace NUnicode { public: TNormalizer() - : Decompositor(*Singleton<NPrivate::TStandartDecompositor<IsCompat>>()) - { - } + : Decompositor(*Singleton<NPrivate::TStandartDecompositor<IsCompat>>()) + { + } TNormalizer(const TDecompositor& decompositor) : Decompositor(decompositor) - { - } + { + } template <class T, typename TCharType> inline void Normalize(const TCharType* begin, const TCharType* end, T& out) { @@ -353,7 +353,7 @@ namespace NUnicode { } } }; -} +} //! decompose utf16 or utf32 string to any container supporting push_back or to T* template <NUnicode::ENormalization Norm, class T, typename TCharType> diff --git a/library/cpp/unicode/normalization/ut/normalization_ut.cpp b/library/cpp/unicode/normalization/ut/normalization_ut.cpp index 54d4940a26..27c6f9f5ea 100644 --- a/library/cpp/unicode/normalization/ut/normalization_ut.cpp +++ b/library/cpp/unicode/normalization/ut/normalization_ut.cpp @@ -5,7 +5,7 @@ #include <library/cpp/unicode/normalization/normalization.h> Y_UNIT_TEST_SUITE(TUnicodeNormalizationTest) { - template <NUnicode::ENormalization NormType> + template <NUnicode::ENormalization NormType> void TestInit() { NUnicode::TNormalizer<NormType> normalizer; TString s("упячка detected"); diff --git a/library/cpp/unicode/punycode/punycode.h b/library/cpp/unicode/punycode/punycode.h index af4acc25c1..7d14d2a794 100644 --- a/library/cpp/unicode/punycode/punycode.h +++ b/library/cpp/unicode/punycode/punycode.h @@ -1,5 +1,5 @@ #pragma once - + #include <util/generic/string.h> #include <util/generic/strbuf.h> #include <util/generic/yexception.h> diff --git a/library/cpp/unicode/punycode/punycode_ut.cpp b/library/cpp/unicode/punycode/punycode_ut.cpp index 97271cf0d8..b0208ad7bf 100644 --- a/library/cpp/unicode/punycode/punycode_ut.cpp +++ b/library/cpp/unicode/punycode/punycode_ut.cpp @@ -39,13 +39,13 @@ Y_UNIT_TEST_SUITE(TPunycodeTest) { UNIT_ASSERT(TestRaw("пример", "e1afmkfd")); { - const wchar16 tmp[] = {0x82, 0x81, 0x80, 0}; - UNIT_ASSERT(PunycodeToWide("abc") == tmp); // "abc" is still valid punycode + const wchar16 tmp[] = {0x82, 0x81, 0x80, 0}; + UNIT_ASSERT(PunycodeToWide("abc") == tmp); // "abc" is still valid punycode } UNIT_ASSERT_EXCEPTION(PunycodeToWide(" "), TPunycodeError); UNIT_ASSERT_EXCEPTION(PunycodeToWide("абвгд"), TPunycodeError); - UNIT_ASSERT_EXCEPTION(PunycodeToWide("-"), TPunycodeError); + UNIT_ASSERT_EXCEPTION(PunycodeToWide("-"), TPunycodeError); { TString longIn; @@ -62,23 +62,23 @@ Y_UNIT_TEST_SUITE(TPunycodeTest) { TString buf1; TUtf16String buf2; //Cerr << "Testing " << utf8 << Endl; - return HostNameToPunycode(unicode) == punycode && HostNameToPunycode(UTF8ToWide(punycode)) == punycode // repeated encoding should give same result - && PunycodeToHostName(punycode) == unicode && CanBePunycodeHostName(punycode) == canBePunycode; + return HostNameToPunycode(unicode) == punycode && HostNameToPunycode(UTF8ToWide(punycode)) == punycode // repeated encoding should give same result + && PunycodeToHostName(punycode) == unicode && CanBePunycodeHostName(punycode) == canBePunycode; } static bool TestForced(const TString& bad) { - return ForceHostNameToPunycode(UTF8ToWide(bad)) == bad && ForcePunycodeToHostName(bad) == UTF8ToWide(bad); + return ForceHostNameToPunycode(UTF8ToWide(bad)) == bad && ForcePunycodeToHostName(bad) == UTF8ToWide(bad); } Y_UNIT_TEST(HostNameEncodeDecode) { UNIT_ASSERT(TestHostName("президент.рф", "xn--d1abbgf6aiiy.xn--p1ai", true)); - UNIT_ASSERT(TestHostName("яндекс.ru", "xn--d1acpjx3f.ru", true)); - UNIT_ASSERT(TestHostName("пример", "xn--e1afmkfd", true)); - UNIT_ASSERT(TestHostName("ascii.test", "ascii.test")); + UNIT_ASSERT(TestHostName("яндекс.ru", "xn--d1acpjx3f.ru", true)); + UNIT_ASSERT(TestHostName("пример", "xn--e1afmkfd", true)); + UNIT_ASSERT(TestHostName("ascii.test", "ascii.test")); UNIT_ASSERT(TestHostName("", "")); UNIT_ASSERT(TestHostName(".", ".")); - UNIT_ASSERT(TestHostName("a.", "a.")); // empty root domain is ok + UNIT_ASSERT(TestHostName("a.", "a.")); // empty root domain is ok UNIT_ASSERT(TestHostName("a.b.c.д.e.f", "a.b.c.xn--d1a.e.f", true)); UNIT_ASSERT(TestHostName("а.б.в.г.д", "xn--80a.xn--90a.xn--b1a.xn--c1a.xn--d1a", true)); @@ -97,7 +97,7 @@ Y_UNIT_TEST_SUITE(TPunycodeTest) { // too long domain label TString bad(500, 'a'); UNIT_ASSERT_EXCEPTION(HostNameToPunycode(UTF8ToWide(bad)), TPunycodeError); - UNIT_ASSERT(TestForced(bad)); // but can decode it + UNIT_ASSERT(TestForced(bad)); // but can decode it } { // already has ACE prefix @@ -121,6 +121,6 @@ Y_UNIT_TEST_SUITE(TPunycodeTest) { UNIT_ASSERT(!CanBePunycodeHostName("яндекс.рф")); // non-ascii UNIT_ASSERT(!CanBePunycodeHostName("яндекс.xn--p1ai")); // non-ascii UNIT_ASSERT(!CanBePunycodeHostName("")); - UNIT_ASSERT(!CanBePunycodeHostName("http://xn--a.b")); // scheme prefix is not detected here + UNIT_ASSERT(!CanBePunycodeHostName("http://xn--a.b")); // scheme prefix is not detected here } } diff --git a/library/cpp/unicode/ya.make b/library/cpp/unicode/ya.make index 4fcd9caacc..8410ab63fd 100644 --- a/library/cpp/unicode/ya.make +++ b/library/cpp/unicode/ya.make @@ -1,4 +1,4 @@ -RECURSE( +RECURSE( folding folding/ut normalization @@ -11,4 +11,4 @@ RECURSE( utf8_char/ut utf8_iter utf8_iter/ut -) +) diff --git a/library/cpp/uri/assign.cpp b/library/cpp/uri/assign.cpp index ae9125c727..25b663b636 100644 --- a/library/cpp/uri/assign.cpp +++ b/library/cpp/uri/assign.cpp @@ -8,418 +8,418 @@ #include <util/memory/tempbuf.h> #include <util/string/cast.h> #include <util/system/yassert.h> -#include <util/system/sys_alloc.h> +#include <util/system/sys_alloc.h> namespace NUri { - TMallocPtr<char> TUri::IDNToAscii(const wchar32* idna) { - // XXX: don't use punycode_encode directly as it doesn't include - // proper stringprep and splitting on dot-equivalent characters - char* buf; - static_assert(sizeof(*idna) == sizeof(ui32), "fixme"); - if (IDNA_SUCCESS != idna_to_ascii_4z((const uint32_t*)idna, &buf, 0)) - buf = nullptr; - return buf; - } - - TMallocPtr<char> TUri::IDNToAscii(const TStringBuf& host, ECharset enc) { - TTempBuf buf(sizeof(wchar32) * (1 + host.length())); - wchar32* wbuf = reinterpret_cast<wchar32*>(buf.Data()); - - const size_t written = NDetail::NBaseOps::Recode(host, wbuf, enc).length(); - wbuf[written] = 0; - - return IDNToAscii(wbuf); - } - - TStringBuf TUri::HostToAscii(TStringBuf host, TMallocPtr<char>& buf, bool hasExtended, bool allowIDN, ECharset enc) { - TStringBuf outhost; // store the result here before returning it, to get RVO - - size_t buflen = 0; - - if (hasExtended && !allowIDN) - return outhost; // definitely can't convert - - // charset-recode: RFC 3986, 3.2.2, requires percent-encoded non-ASCII - // chars in reg-name to be UTF-8 so convert to UTF-8 prior to decoding - const bool recoding = CODES_UTF8 != enc && hasExtended; - if (recoding) { - size_t nrd, nwr; - buflen = host.length() * 4; - buf.Reset(static_cast<char*>(y_allocate(buflen))); - if (RECODE_OK != Recode(enc, CODES_UTF8, host.data(), buf.Get(), host.length(), buflen, nrd, nwr)) - return outhost; - host = TStringBuf(buf.Get(), nwr); - } - - // percent-decode - if (0 == buflen) { - buflen = host.length(); - buf.Reset(static_cast<char*>(y_allocate(buflen))); - } - // decoding shortens so writing over host in buf is OK - TMemoryWriteBuffer out(buf.Get(), buflen); - TEncoder decoder(out, FeatureDecodeANY | FeatureToLower); - const long outFlags = decoder.ReEncode(host); - hasExtended = 0 != (outFlags & FeatureEncodeExtendedASCII); - - // check again - if (hasExtended && !allowIDN) + TMallocPtr<char> TUri::IDNToAscii(const wchar32* idna) { + // XXX: don't use punycode_encode directly as it doesn't include + // proper stringprep and splitting on dot-equivalent characters + char* buf; + static_assert(sizeof(*idna) == sizeof(ui32), "fixme"); + if (IDNA_SUCCESS != idna_to_ascii_4z((const uint32_t*)idna, &buf, 0)) + buf = nullptr; + return buf; + } + + TMallocPtr<char> TUri::IDNToAscii(const TStringBuf& host, ECharset enc) { + TTempBuf buf(sizeof(wchar32) * (1 + host.length())); + wchar32* wbuf = reinterpret_cast<wchar32*>(buf.Data()); + + const size_t written = NDetail::NBaseOps::Recode(host, wbuf, enc).length(); + wbuf[written] = 0; + + return IDNToAscii(wbuf); + } + + TStringBuf TUri::HostToAscii(TStringBuf host, TMallocPtr<char>& buf, bool hasExtended, bool allowIDN, ECharset enc) { + TStringBuf outhost; // store the result here before returning it, to get RVO + + size_t buflen = 0; + + if (hasExtended && !allowIDN) + return outhost; // definitely can't convert + + // charset-recode: RFC 3986, 3.2.2, requires percent-encoded non-ASCII + // chars in reg-name to be UTF-8 so convert to UTF-8 prior to decoding + const bool recoding = CODES_UTF8 != enc && hasExtended; + if (recoding) { + size_t nrd, nwr; + buflen = host.length() * 4; + buf.Reset(static_cast<char*>(y_allocate(buflen))); + if (RECODE_OK != Recode(enc, CODES_UTF8, host.data(), buf.Get(), host.length(), buflen, nrd, nwr)) + return outhost; + host = TStringBuf(buf.Get(), nwr); + } + + // percent-decode + if (0 == buflen) { + buflen = host.length(); + buf.Reset(static_cast<char*>(y_allocate(buflen))); + } + // decoding shortens so writing over host in buf is OK + TMemoryWriteBuffer out(buf.Get(), buflen); + TEncoder decoder(out, FeatureDecodeANY | FeatureToLower); + const long outFlags = decoder.ReEncode(host); + hasExtended = 0 != (outFlags & FeatureEncodeExtendedASCII); + + // check again + if (hasExtended && !allowIDN) return outhost; - host = out.Str(); - - // convert to punycode if needed - if (!hasExtended) { - outhost = host; - return outhost; - } + host = out.Str(); + + // convert to punycode if needed + if (!hasExtended) { + outhost = host; + return outhost; + } + + TMallocPtr<char> puny; + try { + puny = IDNToAscii(host); + } catch (const yexception& /* exc */) { + } + + if (!puny) { + // XXX: try user charset unless UTF8 or converted to it + if (CODES_UTF8 == enc || recoding) + return outhost; + try { + puny = IDNToAscii(host, enc); + } catch (const yexception& /* exc */) { + return outhost; + } + if (!puny) + return outhost; + } + + buf = puny; + outhost = buf.Get(); + + return outhost; + } - TMallocPtr<char> puny; - try { - puny = IDNToAscii(host); - } catch (const yexception& /* exc */) { - } + TStringBuf TUri::HostToAscii(const TStringBuf& host, TMallocPtr<char>& buf, bool allowIDN, ECharset enc) { + // find what we have + long haveFlags = 0; + for (size_t i = 0; i != host.length(); ++i) + haveFlags |= TEncoder::GetFlags(host[i]).FeatFlags; - if (!puny) { - // XXX: try user charset unless UTF8 or converted to it - if (CODES_UTF8 == enc || recoding) - return outhost; - try { - puny = IDNToAscii(host, enc); - } catch (const yexception& /* exc */) { - return outhost; - } - if (!puny) - return outhost; - } + // interested in encoded characters or (if IDN is allowed) extended ascii + TStringBuf outhost; + const bool haveExtended = haveFlags & FeatureEncodeExtendedASCII; - buf = puny; - outhost = buf.Get(); + if (!haveExtended || allowIDN) { + if (!haveExtended && 0 == (haveFlags & FeatureDecodeANY)) + outhost = host; + else + outhost = HostToAscii(host, buf, haveExtended, allowIDN, enc); + } - return outhost; - } + return outhost; + } - TStringBuf TUri::HostToAscii(const TStringBuf& host, TMallocPtr<char>& buf, bool allowIDN, ECharset enc) { - // find what we have - long haveFlags = 0; - for (size_t i = 0; i != host.length(); ++i) - haveFlags |= TEncoder::GetFlags(host[i]).FeatFlags; - - // interested in encoded characters or (if IDN is allowed) extended ascii - TStringBuf outhost; - const bool haveExtended = haveFlags & FeatureEncodeExtendedASCII; - - if (!haveExtended || allowIDN) { - if (!haveExtended && 0 == (haveFlags & FeatureDecodeANY)) - outhost = host; - else - outhost = HostToAscii(host, buf, haveExtended, allowIDN, enc); - } - - return outhost; - } - - static inline bool AppendField(TMemoryWriteBuffer& out, TField::EField fld, const TStringBuf& val, long flags) { + static inline bool AppendField(TMemoryWriteBuffer& out, TField::EField fld, const TStringBuf& val, long flags) { if (val.empty()) - return false; - if (flags & TFeature::FeaturesAllEncoder) - TUri::ReEncodeField(out, val, fld, flags); + return false; + if (flags & TFeature::FeaturesAllEncoder) + TUri::ReEncodeField(out, val, fld, flags); else - out << val; - return true; + out << val; + return true; } - TState::EParsed TUri::AssignImpl(const TParser& parser, TScheme::EKind defscheme) { - Clear(); + TState::EParsed TUri::AssignImpl(const TParser& parser, TScheme::EKind defscheme) { + Clear(); - TState::EParsed ret = parser.State; - if (ParsedBadFormat <= ret) - return ret; + TState::EParsed ret = parser.State; + if (ParsedBadFormat <= ret) + return ret; - const TSection& scheme = parser.Get(FieldScheme); - const TSchemeInfo& schemeInfo = SetSchemeImpl(parser.Scheme); + const TSection& scheme = parser.Get(FieldScheme); + const TSchemeInfo& schemeInfo = SetSchemeImpl(parser.Scheme); - // set the scheme always if available + // set the scheme always if available if (schemeInfo.Str.empty() && scheme.IsSet()) - FldSet(FieldScheme, scheme.Get()); + FldSet(FieldScheme, scheme.Get()); - if (ParsedOK != ret) - return ret; + if (ParsedOK != ret) + return ret; - size_t buflen = 0; + size_t buflen = 0; - // special processing for fields + // special processing for fields - const bool convertIDN = parser.Flags & FeatureConvertHostIDN; - long flags = parser.Flags.Allow; - if (convertIDN) - flags |= FeatureAllowHostIDN | FeatureCheckHost; + const bool convertIDN = parser.Flags & FeatureConvertHostIDN; + long flags = parser.Flags.Allow; + if (convertIDN) + flags |= FeatureAllowHostIDN | FeatureCheckHost; - // process non-ASCII host for punycode + // process non-ASCII host for punycode - TMallocPtr<char> hostptr; - TStringBuf hostascii; // empty: use host field; non-empty: ascii - bool hostConverted = false; // hostascii is empty or the original - const TSection& host = parser.Get(FieldHost); - if (host.IsSet() && !FldIsSet(FieldHost)) { - const bool allowIDN = (flags & FeatureAllowHostIDN); - const TStringBuf hostbuf = host.Get(); + TMallocPtr<char> hostptr; + TStringBuf hostascii; // empty: use host field; non-empty: ascii + bool hostConverted = false; // hostascii is empty or the original + const TSection& host = parser.Get(FieldHost); + if (host.IsSet() && !FldIsSet(FieldHost)) { + const bool allowIDN = (flags & FeatureAllowHostIDN); + const TStringBuf hostbuf = host.Get(); - // if we know we have and allow extended-ASCII chars, no need to check further - if (allowIDN && (host.GetFlagsAllPlaintext() & FeatureEncodeExtendedASCII)) - hostascii = HostToAscii(hostbuf, hostptr, true, true, parser.Enc); - else - hostascii = HostToAscii(hostbuf, hostptr, allowIDN, parser.Enc); + // if we know we have and allow extended-ASCII chars, no need to check further + if (allowIDN && (host.GetFlagsAllPlaintext() & FeatureEncodeExtendedASCII)) + hostascii = HostToAscii(hostbuf, hostptr, true, true, parser.Enc); + else + hostascii = HostToAscii(hostbuf, hostptr, allowIDN, parser.Enc); if (hostascii.empty()) - ret = ParsedBadHost; // exists but cannot be converted - else if (hostbuf.data() != hostascii.data()) { - hostConverted = true; - buflen += 1 + hostascii.length(); - if (convertIDN) - FldMarkSet(FieldHost); // so that we don't process host below - } + ret = ParsedBadHost; // exists but cannot be converted + else if (hostbuf.data() != hostascii.data()) { + hostConverted = true; + buflen += 1 + hostascii.length(); + if (convertIDN) + FldMarkSet(FieldHost); // so that we don't process host below + } + } + + // add unprocessed fields + + for (int idx = 0; idx < FieldUrlMAX; ++idx) { + const EField fld = EField(idx); + const TSection& section = parser.Get(fld); + if (section.IsSet() && !FldIsSet(fld)) + buflen += 1 + section.EncodedLen(); // includes null } + if (0 == buflen) // no more sections set? + return ret; - // add unprocessed fields - - for (int idx = 0; idx < FieldUrlMAX; ++idx) { - const EField fld = EField(idx); - const TSection& section = parser.Get(fld); - if (section.IsSet() && !FldIsSet(fld)) - buflen += 1 + section.EncodedLen(); // includes null - } - if (0 == buflen) // no more sections set? - return ret; - - // process #! fragments - // https://developers.google.com/webmasters/ajax-crawling/docs/specification + // process #! fragments + // https://developers.google.com/webmasters/ajax-crawling/docs/specification static const TStringBuf escFragPrefix(TStringBuf("_escaped_fragment_=")); - bool encHashBangFrag = false; - TStringBuf qryBeforeEscapedFragment; - TStringBuf qryEscapedFragment; - do { - if (FldIsSet(FieldFrag) || FldIsSet(FieldQuery)) - break; - - const TSection& frag = parser.Get(FieldFrag); - if (frag.IsSet()) { - if (0 == (parser.Flags & FeatureHashBangToEscapedFragment)) - break; - const TStringBuf fragbuf = frag.Get(); + bool encHashBangFrag = false; + TStringBuf qryBeforeEscapedFragment; + TStringBuf qryEscapedFragment; + do { + if (FldIsSet(FieldFrag) || FldIsSet(FieldQuery)) + break; + + const TSection& frag = parser.Get(FieldFrag); + if (frag.IsSet()) { + if (0 == (parser.Flags & FeatureHashBangToEscapedFragment)) + break; + const TStringBuf fragbuf = frag.Get(); if (fragbuf.empty() || '!' != fragbuf[0]) - break; - encHashBangFrag = true; - // '!' will make space for '&' or '\0' if needed - buflen += escFragPrefix.length(); - buflen += 2 * fragbuf.length(); // we don't know how many will be encoded - } else { - const TSection& qry = parser.Get(FieldQuery); - if (!qry.IsSet()) - break; - // FeatureHashBangToEscapedFragment has preference - if (FeatureEscapedToHashBangFragment != (parser.Flags & FeaturesEscapedFragment)) - break; - qry.Get().RSplit('&', qryBeforeEscapedFragment, qryEscapedFragment); - if (!qryEscapedFragment.StartsWith(escFragPrefix)) { - qryEscapedFragment.Clear(); - break; - } - qryEscapedFragment.Skip(escFragPrefix.length()); - buflen += 2; // for '!' and '\0' in fragment - buflen -= escFragPrefix.length(); - } - } while (false); - - // now set all fields prior to validating + break; + encHashBangFrag = true; + // '!' will make space for '&' or '\0' if needed + buflen += escFragPrefix.length(); + buflen += 2 * fragbuf.length(); // we don't know how many will be encoded + } else { + const TSection& qry = parser.Get(FieldQuery); + if (!qry.IsSet()) + break; + // FeatureHashBangToEscapedFragment has preference + if (FeatureEscapedToHashBangFragment != (parser.Flags & FeaturesEscapedFragment)) + break; + qry.Get().RSplit('&', qryBeforeEscapedFragment, qryEscapedFragment); + if (!qryEscapedFragment.StartsWith(escFragPrefix)) { + qryEscapedFragment.Clear(); + break; + } + qryEscapedFragment.Skip(escFragPrefix.length()); + buflen += 2; // for '!' and '\0' in fragment + buflen -= escFragPrefix.length(); + } + } while (false); + + // now set all fields prior to validating Alloc(buflen); TMemoryWriteBuffer out(Buffer.data(), Buffer.size()); - for (int idx = 0; idx < FieldUrlMAX; ++idx) { - const EField fld = EField(idx); + for (int idx = 0; idx < FieldUrlMAX; ++idx) { + const EField fld = EField(idx); - const TSection& section = parser.Get(fld); - if (!section.IsSet() || FldIsSet(fld)) - continue; + const TSection& section = parser.Get(fld); + if (!section.IsSet() || FldIsSet(fld)) + continue; - if (FieldQuery == fld && encHashBangFrag) - continue; + if (FieldQuery == fld && encHashBangFrag) + continue; - if (FieldFrag == fld && qryEscapedFragment.IsInited()) - continue; + if (FieldFrag == fld && qryEscapedFragment.IsInited()) + continue; - char* beg = out.Buf(); - TStringBuf val = section.Get(); - long careFlags = section.GetFlagsEncode(); + char* beg = out.Buf(); + TStringBuf val = section.Get(); + long careFlags = section.GetFlagsEncode(); - switch (fld) { - default: - break; + switch (fld) { + default: + break; - case FieldQuery: - if (qryEscapedFragment.IsInited()) { - const EField dstfld = FieldFrag; // that's where we will store - out << '!'; + case FieldQuery: + if (qryEscapedFragment.IsInited()) { + const EField dstfld = FieldFrag; // that's where we will store + out << '!'; if (!qryEscapedFragment.empty()) - ReEncodeToField(out, qryEscapedFragment, fld, FeatureDecodeANY | careFlags, dstfld, FeatureDecodeANY | parser.GetFieldFlags(dstfld)); - FldSetNoDirty(dstfld, TStringBuf(beg, out.Buf())); + ReEncodeToField(out, qryEscapedFragment, fld, FeatureDecodeANY | careFlags, dstfld, FeatureDecodeANY | parser.GetFieldFlags(dstfld)); + FldSetNoDirty(dstfld, TStringBuf(beg, out.Buf())); if (qryBeforeEscapedFragment.empty()) - continue; - out << '\0'; - beg = out.Buf(); - val = qryBeforeEscapedFragment; - } - break; - - case FieldFrag: - if (encHashBangFrag) { - const EField dstfld = FieldQuery; // that's where we will store - const TSection& qry = parser.Get(dstfld); - if (qry.IsSet()) - if (AppendField(out, dstfld, qry.Get(), qry.GetFlagsEncode())) - out << '&'; - out << escFragPrefix; - val.Skip(1); // skip '!' - ReEncodeToField(out, val, fld, careFlags, dstfld, parser.GetFieldFlags(dstfld)); - FldSetNoDirty(dstfld, TStringBuf(beg, out.Buf())); - continue; - } - break; + continue; + out << '\0'; + beg = out.Buf(); + val = qryBeforeEscapedFragment; + } + break; + + case FieldFrag: + if (encHashBangFrag) { + const EField dstfld = FieldQuery; // that's where we will store + const TSection& qry = parser.Get(dstfld); + if (qry.IsSet()) + if (AppendField(out, dstfld, qry.Get(), qry.GetFlagsEncode())) + out << '&'; + out << escFragPrefix; + val.Skip(1); // skip '!' + ReEncodeToField(out, val, fld, careFlags, dstfld, parser.GetFieldFlags(dstfld)); + FldSetNoDirty(dstfld, TStringBuf(beg, out.Buf())); + continue; + } + break; + } + + AppendField(out, fld, val, careFlags); + char* end = out.Buf(); + + if (careFlags & FeaturePathOperation) { + if (!PathOperation(beg, end, PathOperationFlag(parser.Flags))) + return ParsedBadPath; + + Y_ASSERT(beg >= out.Beg()); + out.SetPos(end); } - AppendField(out, fld, val, careFlags); - char* end = out.Buf(); - - if (careFlags & FeaturePathOperation) { - if (!PathOperation(beg, end, PathOperationFlag(parser.Flags))) - return ParsedBadPath; - - Y_ASSERT(beg >= out.Beg()); - out.SetPos(end); - } - - FldSetNoDirty(fld, TStringBuf(beg, end)); - - // special character case - const long checkChars = section.GetFlagsAllPlaintext() & FeaturesCheckSpecialChar; - if (0 != checkChars) { // has unencoded special chars: check permission - const long allowChars = parser.GetFieldFlags(fld) & checkChars; - if (checkChars != allowChars) - ret = ParsedBadFormat; + FldSetNoDirty(fld, TStringBuf(beg, end)); + + // special character case + const long checkChars = section.GetFlagsAllPlaintext() & FeaturesCheckSpecialChar; + if (0 != checkChars) { // has unencoded special chars: check permission + const long allowChars = parser.GetFieldFlags(fld) & checkChars; + if (checkChars != allowChars) + ret = ParsedBadFormat; } out << '\0'; } - if (hostConverted) { - char* beg = out.Buf(); - out << hostascii; - char* end = out.Buf(); - const EField fld = convertIDN ? FieldHost : FieldHostAscii; - FldSetNoDirty(fld, TStringBuf(beg, end)); + if (hostConverted) { + char* beg = out.Buf(); + out << hostascii; + char* end = out.Buf(); + const EField fld = convertIDN ? FieldHost : FieldHostAscii; + FldSetNoDirty(fld, TStringBuf(beg, end)); out << '\0'; - } + } Buffer.Resize(out.Len()); - if (GetScheme() == SchemeEmpty && SchemeEmpty != defscheme) { - if (SchemeUnknown == defscheme) - ret = ParsedBadScheme; - else - SetSchemeImpl(defscheme); + if (GetScheme() == SchemeEmpty && SchemeEmpty != defscheme) { + if (SchemeUnknown == defscheme) + ret = ParsedBadScheme; + else + SetSchemeImpl(defscheme); } - if (0 == (parser.Flags & FeatureAllowEmptyPath)) - CheckMissingFields(); + if (0 == (parser.Flags & FeatureAllowEmptyPath)) + CheckMissingFields(); - const TStringBuf& port = GetField(FieldPort); + const TStringBuf& port = GetField(FieldPort); if (!port.empty()) { - if (!TryFromString<ui16>(port, Port)) - ret = ParsedBadPort; + if (!TryFromString<ui16>(port, Port)) + ret = ParsedBadPort; } - if (ParsedOK != ret) - return ret; + if (ParsedOK != ret) + return ret; - // run validity checks now that all fields are set + // run validity checks now that all fields are set - // check the host for DNS compliance - do { - if (0 == (flags & FeatureCheckHost)) - break; + // check the host for DNS compliance + do { + if (0 == (flags & FeatureCheckHost)) + break; if (hostascii.empty()) - hostascii = GetField(FieldHost); + hostascii = GetField(FieldHost); if (hostascii.empty()) - break; - // IP literal - if ('[' == hostascii[0] && ']' == hostascii.back()) - break; - ret = CheckHost(hostascii); - if (ParsedOK != ret) - return ret; - } while (false); - - return ret; + break; + // IP literal + if ('[' == hostascii[0] && ']' == hostascii.back()) + break; + ret = CheckHost(hostascii); + if (ParsedOK != ret) + return ret; + } while (false); + + return ret; } - TState::EParsed TUri::ParseImpl(const TStringBuf& url, const TParseFlags& flags, ui32 maxlen, TScheme::EKind defscheme, ECharset enc) { - Clear(); + TState::EParsed TUri::ParseImpl(const TStringBuf& url, const TParseFlags& flags, ui32 maxlen, TScheme::EKind defscheme, ECharset enc) { + Clear(); if (url.empty()) - return ParsedEmpty; + return ParsedEmpty; - if (maxlen > 0 && url.length() > maxlen) - return ParsedTooLong; + if (maxlen > 0 && url.length() > maxlen) + return ParsedTooLong; - const TParser parser(flags, url, enc); + const TParser parser(flags, url, enc); - return AssignImpl(parser, defscheme); - } - - TState::EParsed TUri::Parse(const TStringBuf& url, const TParseFlags& flags, const TStringBuf& url_base, ui32 maxlen, ECharset enc) { - const TParseFlags flags1 = flags.Exclude(FeatureNoRelPath); + return AssignImpl(parser, defscheme); + } + + TState::EParsed TUri::Parse(const TStringBuf& url, const TParseFlags& flags, const TStringBuf& url_base, ui32 maxlen, ECharset enc) { + const TParseFlags flags1 = flags.Exclude(FeatureNoRelPath); TState::EParsed ret = ParseImpl(url, url_base.empty() ? flags : flags1, maxlen, SchemeEmpty, enc); if (ParsedOK != ret) return ret; if (!url_base.empty() && !IsValidAbs()) { - TUri base; - ret = base.ParseImpl(url_base, flags, maxlen, SchemeEmpty, enc); - if (ParsedOK != ret) - return ret; - Merge(base, PathOperationFlag(flags)); - } - - Rewrite(); + TUri base; + ret = base.ParseImpl(url_base, flags, maxlen, SchemeEmpty, enc); + if (ParsedOK != ret) + return ret; + Merge(base, PathOperationFlag(flags)); + } + + Rewrite(); return ret; - } + } - TState::EParsed TUri::Parse(const TStringBuf& url, const TUri& base, const TParseFlags& flags, ui32 maxlen, ECharset enc) { - const TState::EParsed ret = ParseImpl(url, flags, maxlen, SchemeEmpty, enc); + TState::EParsed TUri::Parse(const TStringBuf& url, const TUri& base, const TParseFlags& flags, ui32 maxlen, ECharset enc) { + const TState::EParsed ret = ParseImpl(url, flags, maxlen, SchemeEmpty, enc); if (ParsedOK != ret) return ret; - if (!IsValidAbs()) - Merge(base, PathOperationFlag(flags)); + if (!IsValidAbs()) + Merge(base, PathOperationFlag(flags)); - Rewrite(); + Rewrite(); return ret; - } + } - TState::EParsed TUri::ParseAbsUri(const TStringBuf& url, const TParseFlags& flags, ui32 maxlen, TScheme::EKind defscheme, ECharset enc) { - const TState::EParsed ret = ParseImpl( - url, flags | FeatureNoRelPath, maxlen, defscheme, enc); - if (ParsedOK != ret) - return ret; + TState::EParsed TUri::ParseAbsUri(const TStringBuf& url, const TParseFlags& flags, ui32 maxlen, TScheme::EKind defscheme, ECharset enc) { + const TState::EParsed ret = ParseImpl( + url, flags | FeatureNoRelPath, maxlen, defscheme, enc); + if (ParsedOK != ret) + return ret; - if (IsNull(FlagHost)) - return ParsedBadHost; + if (IsNull(FlagHost)) + return ParsedBadHost; - Rewrite(); - return ParsedOK; - } + Rewrite(); + return ParsedOK; + } } diff --git a/library/cpp/uri/common.cpp b/library/cpp/uri/common.cpp index 05af1e57d1..f0419c6ae9 100644 --- a/library/cpp/uri/common.cpp +++ b/library/cpp/uri/common.cpp @@ -1,115 +1,115 @@ #include "common.h" - + #include <util/generic/map.h> -#include <util/generic/singleton.h> - +#include <util/generic/singleton.h> + namespace NUri { - static_assert(TFeature::FeatureMAX <= sizeof(unsigned long) * 8, "expect TFeature::FeatureMAX <= sizeof(unsigned long) * 8"); + static_assert(TFeature::FeatureMAX <= sizeof(unsigned long) * 8, "expect TFeature::FeatureMAX <= sizeof(unsigned long) * 8"); - const TSchemeInfo TSchemeInfo::Registry[] = { - TSchemeInfo(TScheme::SchemeEmpty, TStringBuf()), // scheme is empty and inited + const TSchemeInfo TSchemeInfo::Registry[] = { + TSchemeInfo(TScheme::SchemeEmpty, TStringBuf()), // scheme is empty and inited TSchemeInfo(TScheme::SchemeHTTP, TStringBuf("http"), TField::FlagHost | TField::FlagPath, 80), TSchemeInfo(TScheme::SchemeHTTPS, TStringBuf("https"), TField::FlagHost | TField::FlagPath, 443), TSchemeInfo(TScheme::SchemeFTP, TStringBuf("ftp"), TField::FlagHost | TField::FlagPath, 20), TSchemeInfo(TScheme::SchemeFILE, TStringBuf("file"), TField::FlagPath), TSchemeInfo(TScheme::SchemeWS, TStringBuf("ws"), TField::FlagHost | TField::FlagPath, 80), TSchemeInfo(TScheme::SchemeWSS, TStringBuf("wss"), TField::FlagHost | TField::FlagPath, 443), - // add above - TSchemeInfo(TScheme::SchemeUnknown, TStringBuf()) // scheme is empty and uninited - }; - - namespace { - struct TLessNoCase { - bool operator()(const TStringBuf& lt, const TStringBuf& rt) const { - return 0 > CompareNoCase(lt, rt); - } - }; - - class TSchemeInfoMap { - typedef TMap<TStringBuf, TScheme::EKind, TLessNoCase> TdMap; - TdMap Map_; + // add above + TSchemeInfo(TScheme::SchemeUnknown, TStringBuf()) // scheme is empty and uninited + }; - public: - TSchemeInfoMap() { - for (int i = TScheme::SchemeEmpty; i < TScheme::SchemeUnknown; ++i) { - const TSchemeInfo& info = TSchemeInfo::Get(TScheme::EKind(i)); - Map_.insert(std::make_pair(info.Str, info.Kind)); - } - } + namespace { + struct TLessNoCase { + bool operator()(const TStringBuf& lt, const TStringBuf& rt) const { + return 0 > CompareNoCase(lt, rt); + } + }; - TScheme::EKind Get(const TStringBuf& scheme) const { - const TdMap::const_iterator it = Map_.find(scheme); - return Map_.end() == it ? TScheme::SchemeUnknown : it->second; - } + class TSchemeInfoMap { + typedef TMap<TStringBuf, TScheme::EKind, TLessNoCase> TdMap; + TdMap Map_; - static const TSchemeInfoMap& Instance() { - return *Singleton<TSchemeInfoMap>(); - } - }; + public: + TSchemeInfoMap() { + for (int i = TScheme::SchemeEmpty; i < TScheme::SchemeUnknown; ++i) { + const TSchemeInfo& info = TSchemeInfo::Get(TScheme::EKind(i)); + Map_.insert(std::make_pair(info.Str, info.Kind)); + } + } + TScheme::EKind Get(const TStringBuf& scheme) const { + const TdMap::const_iterator it = Map_.find(scheme); + return Map_.end() == it ? TScheme::SchemeUnknown : it->second; + } + + static const TSchemeInfoMap& Instance() { + return *Singleton<TSchemeInfoMap>(); + } + }; + } - - const TSchemeInfo& TSchemeInfo::Get(const TStringBuf& scheme) { - return Registry[TSchemeInfoMap::Instance().Get(scheme)]; - } - - const char* ParsedStateToString(const TState::EParsed& t) { - switch (t) { - case TState::ParsedOK: - return "ParsedOK"; - case TState::ParsedEmpty: - return "ParsedEmpty"; - case TState::ParsedRootless: - return "ParsedRootless"; - case TState::ParsedBadFormat: - return "ParsedBadFormat"; - case TState::ParsedBadPath: - return "ParsedBadPath"; - case TState::ParsedTooLong: - return "ParsedTooLong"; - case TState::ParsedBadPort: - return "ParsedBadPort"; - case TState::ParsedBadAuth: - return "ParsedBadAuth"; - case TState::ParsedBadScheme: - return "ParsedBadScheme"; - case TState::ParsedBadHost: - return "ParsedBadHost"; - default: - return "Parsed[Unknown]"; - } + + const TSchemeInfo& TSchemeInfo::Get(const TStringBuf& scheme) { + return Registry[TSchemeInfoMap::Instance().Get(scheme)]; } + + const char* ParsedStateToString(const TState::EParsed& t) { + switch (t) { + case TState::ParsedOK: + return "ParsedOK"; + case TState::ParsedEmpty: + return "ParsedEmpty"; + case TState::ParsedRootless: + return "ParsedRootless"; + case TState::ParsedBadFormat: + return "ParsedBadFormat"; + case TState::ParsedBadPath: + return "ParsedBadPath"; + case TState::ParsedTooLong: + return "ParsedTooLong"; + case TState::ParsedBadPort: + return "ParsedBadPort"; + case TState::ParsedBadAuth: + return "ParsedBadAuth"; + case TState::ParsedBadScheme: + return "ParsedBadScheme"; + case TState::ParsedBadHost: + return "ParsedBadHost"; + default: + return "Parsed[Unknown]"; + } + } - const char* FieldToString(const TField::EField& t) { - switch (t) { - case TField::FieldScheme: - return "scheme"; - case TField::FieldUser: - return "username"; - case TField::FieldPass: - return "password"; - case TField::FieldHost: - return "host"; - case TField::FieldHostAscii: - return "hostascii"; - case TField::FieldPort: - return "port"; - case TField::FieldPath: - return "path"; - case TField::FieldQuery: - return "query"; - case TField::FieldFrag: - return "fragment"; - default: - return "Field[Unknown]"; - } + const char* FieldToString(const TField::EField& t) { + switch (t) { + case TField::FieldScheme: + return "scheme"; + case TField::FieldUser: + return "username"; + case TField::FieldPass: + return "password"; + case TField::FieldHost: + return "host"; + case TField::FieldHostAscii: + return "hostascii"; + case TField::FieldPort: + return "port"; + case TField::FieldPath: + return "path"; + case TField::FieldQuery: + return "query"; + case TField::FieldFrag: + return "fragment"; + default: + return "Field[Unknown]"; + } } - const char* SchemeKindToString(const TScheme::EKind& t) { - const TSchemeInfo& info = TSchemeInfo::Get(t); + const char* SchemeKindToString(const TScheme::EKind& t) { + const TSchemeInfo& info = TSchemeInfo::Get(t); if (!info.Str.empty()) - return info.Str.data(); - return TScheme::SchemeEmpty == t ? "empty" : "unknown"; + return info.Str.data(); + return TScheme::SchemeEmpty == t ? "empty" : "unknown"; } } diff --git a/library/cpp/uri/common.h b/library/cpp/uri/common.h index 8025357763..bd1aca3318 100644 --- a/library/cpp/uri/common.h +++ b/library/cpp/uri/common.h @@ -2,237 +2,237 @@ #include <util/stream/output.h> #include <util/system/compat.h> -#include <util/generic/strbuf.h> - +#include <util/generic/strbuf.h> + namespace NUri { - namespace NEncode { - class TEncoder; - class TEncodeMapperBase; - struct TCharFlags; - } - - namespace NParse { - class TRange; - } - - class TParser; - - struct TField { -#define FIELD_NAME(f) Field##f -#define FIELD_FLAG(f) Flag##f = 1U << FIELD_NAME(f) - - enum EField { - FIELD_NAME(Scheme), - FIELD_NAME(User), - FIELD_NAME(Pass), - FIELD_NAME(Host), - FIELD_NAME(Port), - FIELD_NAME(Path), - FIELD_NAME(Query), - FIELD_NAME(Frag), - - // add fields above - FieldUrlMAX, - // reset count so actual field offsets are not interrupted - FieldUrlLast = FieldUrlMAX - 1, - // add extra fields below - - FIELD_NAME(HostAscii), - - // add extra fields above - FieldAllMAX, - // add aliases below - - FieldUsername = FieldUser, - FieldPassword = FieldPass, - FieldFragment = FieldFrag, - }; - - enum EFlags { - FIELD_FLAG(Scheme), - FIELD_FLAG(User), - FIELD_FLAG(Pass), - FIELD_FLAG(Host), - FIELD_FLAG(Port), - FIELD_FLAG(Path), - FIELD_FLAG(Query), - FIELD_FLAG(Frag), - FIELD_FLAG(UrlMAX), - FIELD_FLAG(HostAscii), - FIELD_FLAG(AllMAX), - - FlagHostPort = FlagHost | FlagPort, - FlagAuth = FlagUser | FlagPass, - FlagFragment = FlagFrag, - FlagAction = FlagScheme | FlagHostPort | FlagPath, - FlagNoFrag = FlagAction | FlagQuery, - FlagUrlFields = FlagUrlMAX - 1, - FlagAll = FlagUrlFields, // obsolete, for backwards compatibility - FlagAllFields = FlagAllMAX - 1 - }; + namespace NEncode { + class TEncoder; + class TEncodeMapperBase; + struct TCharFlags; + } + + namespace NParse { + class TRange; + } + + class TParser; + + struct TField { +#define FIELD_NAME(f) Field##f +#define FIELD_FLAG(f) Flag##f = 1U << FIELD_NAME(f) + + enum EField { + FIELD_NAME(Scheme), + FIELD_NAME(User), + FIELD_NAME(Pass), + FIELD_NAME(Host), + FIELD_NAME(Port), + FIELD_NAME(Path), + FIELD_NAME(Query), + FIELD_NAME(Frag), + + // add fields above + FieldUrlMAX, + // reset count so actual field offsets are not interrupted + FieldUrlLast = FieldUrlMAX - 1, + // add extra fields below + + FIELD_NAME(HostAscii), + + // add extra fields above + FieldAllMAX, + // add aliases below + + FieldUsername = FieldUser, + FieldPassword = FieldPass, + FieldFragment = FieldFrag, + }; + + enum EFlags { + FIELD_FLAG(Scheme), + FIELD_FLAG(User), + FIELD_FLAG(Pass), + FIELD_FLAG(Host), + FIELD_FLAG(Port), + FIELD_FLAG(Path), + FIELD_FLAG(Query), + FIELD_FLAG(Frag), + FIELD_FLAG(UrlMAX), + FIELD_FLAG(HostAscii), + FIELD_FLAG(AllMAX), + + FlagHostPort = FlagHost | FlagPort, + FlagAuth = FlagUser | FlagPass, + FlagFragment = FlagFrag, + FlagAction = FlagScheme | FlagHostPort | FlagPath, + FlagNoFrag = FlagAction | FlagQuery, + FlagUrlFields = FlagUrlMAX - 1, + FlagAll = FlagUrlFields, // obsolete, for backwards compatibility + FlagAllFields = FlagAllMAX - 1 + }; #undef FIELD_NAME #undef FIELD_FLAG + }; + + struct TState { + enum EParsed { + ParsedOK = 0, + ParsedEmpty = 1, + ParsedOpaque = 2, + ParsedRootless = ParsedOpaque, + ParsedBadFormat, // must follow all non-error states immediately + ParsedBadPath, + ParsedTooLong, + ParsedBadPort, + ParsedBadAuth, + ParsedBadScheme, + ParsedBadHost, + + // add before this line + ParsedMAX + }; }; - struct TState { - enum EParsed { - ParsedOK = 0, - ParsedEmpty = 1, - ParsedOpaque = 2, - ParsedRootless = ParsedOpaque, - ParsedBadFormat, // must follow all non-error states immediately - ParsedBadPath, - ParsedTooLong, - ParsedBadPort, - ParsedBadAuth, - ParsedBadScheme, - ParsedBadHost, - - // add before this line - ParsedMAX - }; - }; - - struct TScheme { - // don't forget to define a SchemeRegistry entry - enum EKind { - SchemeEmpty - // add schemes below this line - , - SchemeHTTP, - SchemeHTTPS, - SchemeFTP, + struct TScheme { + // don't forget to define a SchemeRegistry entry + enum EKind { + SchemeEmpty + // add schemes below this line + , + SchemeHTTP, + SchemeHTTPS, + SchemeFTP, SchemeFILE, SchemeWS, SchemeWSS - // add schemes above this line - , - SchemeUnknown - }; + // add schemes above this line + , + SchemeUnknown + }; }; - class TFeature { - friend class NEncode::TEncoder; - friend class NEncode::TEncodeMapperBase; - friend struct NEncode::TCharFlags; - friend class TParser; - friend class NParse::TRange; + class TFeature { + friend class NEncode::TEncoder; + friend class NEncode::TEncodeMapperBase; + friend struct NEncode::TCharFlags; + friend class TParser; + friend class NParse::TRange; -#define FEATURE_NAME(f) _BitFeature##f -#define FEATURE_FLAG_NAME(f) Feature##f +#define FEATURE_NAME(f) _BitFeature##f +#define FEATURE_FLAG_NAME(f) Feature##f #define FEATURE_FLAG(f) FEATURE_FLAG_NAME(f) = 1UL << FEATURE_NAME(f) - protected: - enum EBit { - //============================== - // Cases interpreted as errors: - //============================== + protected: + enum EBit { + //============================== + // Cases interpreted as errors: + //============================== - // allows authorization user/password in URL - FEATURE_NAME(AuthSupported), + // allows authorization user/password in URL + FEATURE_NAME(AuthSupported), - // allows all known schemes in URL - FEATURE_NAME(SchemeKnown), + // allows all known schemes in URL + FEATURE_NAME(SchemeKnown), - // allows all schemes, not only known - FEATURE_NAME(SchemeFlexible), + // allows all schemes, not only known + FEATURE_NAME(SchemeFlexible), - // allow opaque (RFC 2396) or rootless (RFC 3986) urls - FEATURE_NAME(AllowRootless), + // allow opaque (RFC 2396) or rootless (RFC 3986) urls + FEATURE_NAME(AllowRootless), - //============================== - // Cases interpreted for processing (if required): - // (effects on result of Parse method) - //============================== + //============================== + // Cases interpreted for processing (if required): + // (effects on result of Parse method) + //============================== - // path needs normalization - // (simplification of directory tree: /../, /./, etc. - FEATURE_NAME(PathOperation), + // path needs normalization + // (simplification of directory tree: /../, /./, etc. + FEATURE_NAME(PathOperation), - // don't force empty path to "/" - FEATURE_NAME(AllowEmptyPath), + // don't force empty path to "/" + FEATURE_NAME(AllowEmptyPath), - // in scheme and host segments: - // change upper case letters onto lower case ones - FEATURE_NAME(ToLower), + // in scheme and host segments: + // change upper case letters onto lower case ones + FEATURE_NAME(ToLower), - // decode unreserved symbols - FEATURE_NAME(DecodeUnreserved), + // decode unreserved symbols + FEATURE_NAME(DecodeUnreserved), - // legacy: decode standard symbols which may be safe for some fields - FEATURE_NAME(DecodeStandardExtra), + // legacy: decode standard symbols which may be safe for some fields + FEATURE_NAME(DecodeStandardExtra), - // decode symbols allowed (not necessarily safe to decode) only for a given field - // (do not use directly, instead use FeatureDecodeSafe mask below) - FEATURE_NAME(DecodeFieldAllowed), + // decode symbols allowed (not necessarily safe to decode) only for a given field + // (do not use directly, instead use FeatureDecodeSafe mask below) + FEATURE_NAME(DecodeFieldAllowed), - // handling of spaces - FEATURE_NAME(EncodeSpace), + // handling of spaces + FEATURE_NAME(EncodeSpace), - // in query segment: change escaped space to '+' - FEATURE_NAME(EncodeSpaceAsPlus), + // in query segment: change escaped space to '+' + FEATURE_NAME(EncodeSpaceAsPlus), - // escape all string 'markup' symbols - FEATURE_NAME(EncodeForSQL), + // escape all string 'markup' symbols + FEATURE_NAME(EncodeForSQL), - // encoding of extended ascii symbols (8-bit) - FEATURE_NAME(EncodeExtendedASCII), + // encoding of extended ascii symbols (8-bit) + FEATURE_NAME(EncodeExtendedASCII), - // decoding of extended ascii symbols (8-bit) - FEATURE_NAME(DecodeExtendedASCII), + // decoding of extended ascii symbols (8-bit) + FEATURE_NAME(DecodeExtendedASCII), - // encoding of extended delimiter set - FEATURE_NAME(EncodeExtendedDelim), + // encoding of extended delimiter set + FEATURE_NAME(EncodeExtendedDelim), - // decoding of extended delimiter set - FEATURE_NAME(DecodeExtendedDelim), + // decoding of extended delimiter set + FEATURE_NAME(DecodeExtendedDelim), - // control characters [0x00 .. 0x20) - FEATURE_NAME(EncodeCntrl), + // control characters [0x00 .. 0x20) + FEATURE_NAME(EncodeCntrl), - // raw percent character - FEATURE_NAME(EncodePercent), + // raw percent character + FEATURE_NAME(EncodePercent), - // hash fragments - // https://developers.google.com/webmasters/ajax-crawling/docs/specification - // move and encode #! fragments to the query - FEATURE_NAME(HashBangToEscapedFragment), - // move and decode _escaped_fragment_ to the fragment - FEATURE_NAME(EscapedToHashBangFragment), + // hash fragments + // https://developers.google.com/webmasters/ajax-crawling/docs/specification + // move and encode #! fragments to the query + FEATURE_NAME(HashBangToEscapedFragment), + // move and decode _escaped_fragment_ to the fragment + FEATURE_NAME(EscapedToHashBangFragment), - // reject absolute paths started by "/../" - FEATURE_NAME(PathDenyRootParent), + // reject absolute paths started by "/../" + FEATURE_NAME(PathDenyRootParent), - // paths started by "/../" - ignore head - FEATURE_NAME(PathStripRootParent), + // paths started by "/../" - ignore head + FEATURE_NAME(PathStripRootParent), - // tries to fix errors (in particular, in fragment) - FEATURE_NAME(TryToFix), + // tries to fix errors (in particular, in fragment) + FEATURE_NAME(TryToFix), - // check host for DNS compliance - FEATURE_NAME(CheckHost), + // check host for DNS compliance + FEATURE_NAME(CheckHost), - // allow IDN hosts - // host is converted to punycode and stored in FieldHostAscii - // @note host contains characters in the charset of the document - // and percent-encoded characters in UTF-8 (RFC 3986, 3.2.2) - // @note if host contains no extended-ASCII characters and after - // percent-decoding cannot be converted from UTF-8 to UCS-4, - // try to recode from the document charset (if not UTF-8) - FEATURE_NAME(AllowHostIDN), + // allow IDN hosts + // host is converted to punycode and stored in FieldHostAscii + // @note host contains characters in the charset of the document + // and percent-encoded characters in UTF-8 (RFC 3986, 3.2.2) + // @note if host contains no extended-ASCII characters and after + // percent-decoding cannot be converted from UTF-8 to UCS-4, + // try to recode from the document charset (if not UTF-8) + FEATURE_NAME(AllowHostIDN), - // forces AllowHostIDN, but host is replaced with punycode - // forces CheckHost since this replacement is irreversible - FEATURE_NAME(ConvertHostIDN), + // forces AllowHostIDN, but host is replaced with punycode + // forces CheckHost since this replacement is irreversible + FEATURE_NAME(ConvertHostIDN), - // robot interpreted network paths as BadFormat urls - FEATURE_NAME(DenyNetworkPath), + // robot interpreted network paths as BadFormat urls + FEATURE_NAME(DenyNetworkPath), - // robot interprets URLs without a host as BadFormat - FEATURE_NAME(RemoteOnly), + // robot interprets URLs without a host as BadFormat + FEATURE_NAME(RemoteOnly), - /* non-RFC use case: + /* non-RFC use case: * 1. do not allow relative-path-only URIs when they can conflict with * "host/path" (that is, only "./path" or "../path" are allowed); * 2. if neither scheme nor userinfo are present but port is, it must @@ -243,269 +243,269 @@ namespace NUri { * "scheme:pa@th" over "user:pass@host", and even "host:port" when * host contains only scheme-legal characters. */ - FEATURE_NAME(NoRelPath), - - // standard prefers that all hex escapes were using uppercase A-F - FEATURE_NAME(UpperEncoded), - - // internal usage: decode all encoded symbols - FEATURE_NAME(DecodeANY), - - // add before this line - _FeatureMAX - }; - - protected: - enum EPrivate : ui32 { - FEATURE_FLAG(DecodeANY), - FEATURE_FLAG(DecodeFieldAllowed), - FEATURE_FLAG(DecodeStandardExtra), - }; - - public: - enum EPublic : ui32 { - FeatureMAX = _FeatureMAX, - FEATURE_FLAG(AuthSupported), - FEATURE_FLAG(SchemeKnown), - FEATURE_FLAG(SchemeFlexible), - FEATURE_FLAG(AllowRootless), - FEATURE_FLAG_NAME(AllowOpaque) = FEATURE_FLAG_NAME(AllowRootless), - FEATURE_FLAG(PathOperation), - FEATURE_FLAG(AllowEmptyPath), - FEATURE_FLAG(ToLower), - FEATURE_FLAG(DecodeUnreserved), - FEATURE_FLAG(EncodeSpace), - FEATURE_FLAG(EncodeSpaceAsPlus), - FEATURE_FLAG(EncodeForSQL), - FEATURE_FLAG(EncodeExtendedASCII), - FEATURE_FLAG(DecodeExtendedASCII), - FEATURE_FLAG(EncodeExtendedDelim), - FEATURE_FLAG(DecodeExtendedDelim), - FEATURE_FLAG(EncodeCntrl), - FEATURE_FLAG(EncodePercent), - FEATURE_FLAG(HashBangToEscapedFragment), - FEATURE_FLAG(EscapedToHashBangFragment), - FEATURE_FLAG(PathDenyRootParent), - FEATURE_FLAG(PathStripRootParent), - FEATURE_FLAG(TryToFix), - FEATURE_FLAG(CheckHost), - FEATURE_FLAG(AllowHostIDN), - FEATURE_FLAG(ConvertHostIDN), - FEATURE_FLAG(DenyNetworkPath), - FEATURE_FLAG(RemoteOnly), - FEATURE_FLAG(NoRelPath), - FEATURE_FLAG_NAME(HierURI) = FEATURE_FLAG_NAME(NoRelPath), - FEATURE_FLAG(UpperEncoded), - }; + FEATURE_NAME(NoRelPath), + + // standard prefers that all hex escapes were using uppercase A-F + FEATURE_NAME(UpperEncoded), + + // internal usage: decode all encoded symbols + FEATURE_NAME(DecodeANY), + + // add before this line + _FeatureMAX + }; + + protected: + enum EPrivate : ui32 { + FEATURE_FLAG(DecodeANY), + FEATURE_FLAG(DecodeFieldAllowed), + FEATURE_FLAG(DecodeStandardExtra), + }; + + public: + enum EPublic : ui32 { + FeatureMAX = _FeatureMAX, + FEATURE_FLAG(AuthSupported), + FEATURE_FLAG(SchemeKnown), + FEATURE_FLAG(SchemeFlexible), + FEATURE_FLAG(AllowRootless), + FEATURE_FLAG_NAME(AllowOpaque) = FEATURE_FLAG_NAME(AllowRootless), + FEATURE_FLAG(PathOperation), + FEATURE_FLAG(AllowEmptyPath), + FEATURE_FLAG(ToLower), + FEATURE_FLAG(DecodeUnreserved), + FEATURE_FLAG(EncodeSpace), + FEATURE_FLAG(EncodeSpaceAsPlus), + FEATURE_FLAG(EncodeForSQL), + FEATURE_FLAG(EncodeExtendedASCII), + FEATURE_FLAG(DecodeExtendedASCII), + FEATURE_FLAG(EncodeExtendedDelim), + FEATURE_FLAG(DecodeExtendedDelim), + FEATURE_FLAG(EncodeCntrl), + FEATURE_FLAG(EncodePercent), + FEATURE_FLAG(HashBangToEscapedFragment), + FEATURE_FLAG(EscapedToHashBangFragment), + FEATURE_FLAG(PathDenyRootParent), + FEATURE_FLAG(PathStripRootParent), + FEATURE_FLAG(TryToFix), + FEATURE_FLAG(CheckHost), + FEATURE_FLAG(AllowHostIDN), + FEATURE_FLAG(ConvertHostIDN), + FEATURE_FLAG(DenyNetworkPath), + FEATURE_FLAG(RemoteOnly), + FEATURE_FLAG(NoRelPath), + FEATURE_FLAG_NAME(HierURI) = FEATURE_FLAG_NAME(NoRelPath), + FEATURE_FLAG(UpperEncoded), + }; #undef FEATURE_NAME #undef FEATURE_FLAG - public: - //============================== - enum ESets { - // these are guaranteed and will change buffer size + public: + //============================== + enum ESets { + // these are guaranteed and will change buffer size - FeatureDecodeStandard = 0 | FeatureDecodeUnreserved | FeatureDecodeStandardExtra, + FeatureDecodeStandard = 0 | FeatureDecodeUnreserved | FeatureDecodeStandardExtra, - FeaturesDecodeExtended = 0 | FeatureDecodeExtendedASCII | FeatureDecodeExtendedDelim, + FeaturesDecodeExtended = 0 | FeatureDecodeExtendedASCII | FeatureDecodeExtendedDelim, - FeaturesDecode = 0 | FeatureDecodeUnreserved | FeatureDecodeStandard | FeaturesDecodeExtended, + FeaturesDecode = 0 | FeatureDecodeUnreserved | FeatureDecodeStandard | FeaturesDecodeExtended, - FeaturesEncodeExtended = 0 | FeatureEncodeExtendedASCII | FeatureEncodeExtendedDelim, + FeaturesEncodeExtended = 0 | FeatureEncodeExtendedASCII | FeatureEncodeExtendedDelim, - FeaturesEncode = 0 | FeatureEncodeForSQL | FeatureEncodeSpace | FeatureEncodeCntrl | FeatureEncodePercent | FeaturesEncodeExtended, + FeaturesEncode = 0 | FeatureEncodeForSQL | FeatureEncodeSpace | FeatureEncodeCntrl | FeatureEncodePercent | FeaturesEncodeExtended, - // these are not guaranteed to apply to a given field + // these are not guaranteed to apply to a given field - FeatureDecodeAllowed = 0 | FeatureDecodeUnreserved | FeatureDecodeFieldAllowed, + FeatureDecodeAllowed = 0 | FeatureDecodeUnreserved | FeatureDecodeFieldAllowed, - FeaturesMaybeDecode = 0 | FeaturesDecode | FeatureDecodeAllowed, + FeaturesMaybeDecode = 0 | FeaturesDecode | FeatureDecodeAllowed, - FeaturesMaybeEncode = 0 | FeaturesEncode, + FeaturesMaybeEncode = 0 | FeaturesEncode, - FeaturesEncodeDecode = 0 | FeaturesMaybeEncode | FeaturesMaybeDecode, + FeaturesEncodeDecode = 0 | FeaturesMaybeEncode | FeaturesMaybeDecode, - FeaturesAllEncoder = 0 | FeaturesEncodeDecode | FeatureDecodeANY | FeatureToLower | FeatureUpperEncoded | FeatureEncodeSpaceAsPlus, + FeaturesAllEncoder = 0 | FeaturesEncodeDecode | FeatureDecodeANY | FeatureToLower | FeatureUpperEncoded | FeatureEncodeSpaceAsPlus, - //============================== - FeaturesNormalizeSet = 0 | FeaturePathOperation | FeatureToLower | FeatureDecodeAllowed | FeatureEncodeSpaceAsPlus | FeatureEncodeForSQL | FeaturePathStripRootParent | FeatureTryToFix | FeatureUpperEncoded, + //============================== + FeaturesNormalizeSet = 0 | FeaturePathOperation | FeatureToLower | FeatureDecodeAllowed | FeatureEncodeSpaceAsPlus | FeatureEncodeForSQL | FeaturePathStripRootParent | FeatureTryToFix | FeatureUpperEncoded, - FeaturesDefault = 0 // it reproduces old parsedURL - | FeaturePathOperation | FeaturePathDenyRootParent | FeatureCheckHost, + FeaturesDefault = 0 // it reproduces old parsedURL + | FeaturePathOperation | FeaturePathDenyRootParent | FeatureCheckHost, - // essentially allows all valid RFC urls and keeps them as-is - FeaturesBare = 0 | FeatureAuthSupported | FeatureSchemeFlexible | FeatureAllowEmptyPath, + // essentially allows all valid RFC urls and keeps them as-is + FeaturesBare = 0 | FeatureAuthSupported | FeatureSchemeFlexible | FeatureAllowEmptyPath, - FeaturesAll = 0 | FeatureAuthSupported | FeatureSchemeFlexible | FeatureCheckHost | FeaturesNormalizeSet, + FeaturesAll = 0 | FeatureAuthSupported | FeatureSchemeFlexible | FeatureCheckHost | FeaturesNormalizeSet, - // Deprecated, use FeaturesRecommended - FeaturesRobotOld = 0 - // http://tools.ietf.org/html/rfc3986#section-6.2.2 - | FeatureToLower // 6.2.2.1 - | FeatureUpperEncoded // 6.2.2.1 - | FeatureDecodeUnreserved // 6.2.2.2 - | FeaturePathOperation // 6.2.2.3 - | FeaturePathDenyRootParent | FeatureSchemeKnown | FeatureConvertHostIDN | FeatureRemoteOnly | FeatureHashBangToEscapedFragment | FeatureCheckHost, + // Deprecated, use FeaturesRecommended + FeaturesRobotOld = 0 + // http://tools.ietf.org/html/rfc3986#section-6.2.2 + | FeatureToLower // 6.2.2.1 + | FeatureUpperEncoded // 6.2.2.1 + | FeatureDecodeUnreserved // 6.2.2.2 + | FeaturePathOperation // 6.2.2.3 + | FeaturePathDenyRootParent | FeatureSchemeKnown | FeatureConvertHostIDN | FeatureRemoteOnly | FeatureHashBangToEscapedFragment | FeatureCheckHost, - // these are mutually exclusive - FeaturesPath = 0 | FeaturePathDenyRootParent | FeaturePathStripRootParent, + // these are mutually exclusive + FeaturesPath = 0 | FeaturePathDenyRootParent | FeaturePathStripRootParent, - FeaturesEscapedFragment = 0 | FeatureEscapedToHashBangFragment | FeatureHashBangToEscapedFragment, + FeaturesEscapedFragment = 0 | FeatureEscapedToHashBangFragment | FeatureHashBangToEscapedFragment, - FeaturesCheckSpecialChar = 0 | FeatureEncodeSpace | FeatureEncodeCntrl | FeatureEncodePercent, + FeaturesCheckSpecialChar = 0 | FeatureEncodeSpace | FeatureEncodeCntrl | FeatureEncodePercent, - FeaturesEncodePChar = 0 | FeatureUpperEncoded | FeaturesEncodeDecode | FeaturesCheckSpecialChar, + FeaturesEncodePChar = 0 | FeatureUpperEncoded | FeaturesEncodeDecode | FeaturesCheckSpecialChar, - // http://wiki.yandex-team.ru/robot/newDesign/dups/normolization - FeaturesRecommended = 0 | FeatureSchemeKnown | FeatureRemoteOnly | FeatureToLower | FeatureCheckHost | FeatureConvertHostIDN | FeatureHashBangToEscapedFragment | FeatureEncodeSpace | FeatureEncodeCntrl | FeatureEncodeExtendedASCII | FeatureUpperEncoded | FeatureDecodeUnreserved | FeaturePathOperation | FeaturePathStripRootParent, + // http://wiki.yandex-team.ru/robot/newDesign/dups/normolization + FeaturesRecommended = 0 | FeatureSchemeKnown | FeatureRemoteOnly | FeatureToLower | FeatureCheckHost | FeatureConvertHostIDN | FeatureHashBangToEscapedFragment | FeatureEncodeSpace | FeatureEncodeCntrl | FeatureEncodeExtendedASCII | FeatureUpperEncoded | FeatureDecodeUnreserved | FeaturePathOperation | FeaturePathStripRootParent, - FeaturesRobot = FeaturesRecommended - }; + FeaturesRobot = FeaturesRecommended + }; }; - static inline int strnicmp(const char* lt, const char* rt, size_t len) { - return lt == rt ? 0 : ::strnicmp(lt, rt, len); - } + static inline int strnicmp(const char* lt, const char* rt, size_t len) { + return lt == rt ? 0 : ::strnicmp(lt, rt, len); + } - static inline int CompareNoCasePrefix(const TStringBuf& lt, const TStringBuf& rt) { - return strnicmp(lt.data(), rt.data(), rt.length()); - } + static inline int CompareNoCasePrefix(const TStringBuf& lt, const TStringBuf& rt) { + return strnicmp(lt.data(), rt.data(), rt.length()); + } - static inline bool EqualNoCase(const TStringBuf& lt, const TStringBuf& rt) { - return lt.length() == rt.length() && 0 == CompareNoCasePrefix(lt, rt); + static inline bool EqualNoCase(const TStringBuf& lt, const TStringBuf& rt) { + return lt.length() == rt.length() && 0 == CompareNoCasePrefix(lt, rt); } - static inline int CompareNoCase(const TStringBuf& lt, const TStringBuf& rt) { - if (lt.length() == rt.length()) - return CompareNoCasePrefix(lt, rt); - return lt.length() < rt.length() ? -1 : 1; + static inline int CompareNoCase(const TStringBuf& lt, const TStringBuf& rt) { + if (lt.length() == rt.length()) + return CompareNoCasePrefix(lt, rt); + return lt.length() < rt.length() ? -1 : 1; } - class TSchemeInfo { - public: - const TScheme::EKind Kind; - const ui16 Port; - const TStringBuf Str; - const ui32 FldReq; - TSchemeInfo(TScheme::EKind kind, TStringBuf str, ui32 fldReq = 0, ui16 port = 0) - : Kind(kind) - , Port(port) - , Str(str) - , FldReq(fldReq) - { - } - bool Matches(const TStringBuf& scheme) const { - return EqualNoCase(scheme, Str); - } - - public: - static const TSchemeInfo& Get(const TStringBuf& scheme); - static const TSchemeInfo& Get(TScheme::EKind scheme) { - return Registry[scheme]; - } - static TScheme::EKind GetKind(const TStringBuf& scheme) { - return Get(scheme).Kind; - } - static TStringBuf GetCanon(TScheme::EKind scheme) { - return Get(scheme).Str; - } - static ui16 GetDefaultPort(TScheme::EKind scheme) { - return Get(scheme).Port; - } - - private: - static const TSchemeInfo Registry[]; - }; - - struct TParseFlags { - const ui64 Allow; - const ui64 Extra; - TParseFlags(ui64 allow = 0, ui64 extra = 0) - : Allow(allow) - , Extra(extra) - { - } - ui64 operator&(const TParseFlags& flags) const { - return (Allow & flags.Allow) | (Extra & flags.Extra); - } - ui64 operator&(ui64 flags) const { - return (Allow & flags); - } - TParseFlags operator|(const TParseFlags& flags) const { - return TParseFlags(Allow | flags.Allow, Extra | flags.Extra); - } - TParseFlags Exclude(ui64 flags) const { - return TParseFlags(Allow & ~flags, Extra & ~flags); - } - }; - -#define FEATURE_NAME(f) _BitFeature##f -#define FEATURE_FLAG_NAME(f) Feature##f + class TSchemeInfo { + public: + const TScheme::EKind Kind; + const ui16 Port; + const TStringBuf Str; + const ui32 FldReq; + TSchemeInfo(TScheme::EKind kind, TStringBuf str, ui32 fldReq = 0, ui16 port = 0) + : Kind(kind) + , Port(port) + , Str(str) + , FldReq(fldReq) + { + } + bool Matches(const TStringBuf& scheme) const { + return EqualNoCase(scheme, Str); + } + + public: + static const TSchemeInfo& Get(const TStringBuf& scheme); + static const TSchemeInfo& Get(TScheme::EKind scheme) { + return Registry[scheme]; + } + static TScheme::EKind GetKind(const TStringBuf& scheme) { + return Get(scheme).Kind; + } + static TStringBuf GetCanon(TScheme::EKind scheme) { + return Get(scheme).Str; + } + static ui16 GetDefaultPort(TScheme::EKind scheme) { + return Get(scheme).Port; + } + + private: + static const TSchemeInfo Registry[]; + }; + + struct TParseFlags { + const ui64 Allow; + const ui64 Extra; + TParseFlags(ui64 allow = 0, ui64 extra = 0) + : Allow(allow) + , Extra(extra) + { + } + ui64 operator&(const TParseFlags& flags) const { + return (Allow & flags.Allow) | (Extra & flags.Extra); + } + ui64 operator&(ui64 flags) const { + return (Allow & flags); + } + TParseFlags operator|(const TParseFlags& flags) const { + return TParseFlags(Allow | flags.Allow, Extra | flags.Extra); + } + TParseFlags Exclude(ui64 flags) const { + return TParseFlags(Allow & ~flags, Extra & ~flags); + } + }; + +#define FEATURE_NAME(f) _BitFeature##f +#define FEATURE_FLAG_NAME(f) Feature##f #define FEATURE_FLAG(f) FEATURE_FLAG_NAME(f) = 1UL << FEATURE_NAME(f) - struct TQueryArg { - TStringBuf Name; - TStringBuf Value; - - private: - enum EBit { - FEATURE_NAME(Filter), - FEATURE_NAME(SortByName), - FEATURE_NAME(RemoveEmptyQuery), - FEATURE_NAME(RewriteDirty), - _FeatureMAX - }; - - public: - enum EPublic : ui32 { - FeatureMAX = _FeatureMAX, - FEATURE_FLAG(Filter), - FEATURE_FLAG(SortByName), - FEATURE_FLAG(RemoveEmptyQuery), - FEATURE_FLAG(RewriteDirty), - }; - - enum EProcessed { - // OK and clean. - ProcessedOK = 0, - - // OK, but query stored in internal buffer and TUri::Rewrite() is required. - ProcessedDirty = 1, - - ProcessedMalformed = 2, - ProcessedTooMany = 3, - }; - }; - - typedef bool (*TQueryArgFilter)(const TQueryArg& arg, void* filterData); + struct TQueryArg { + TStringBuf Name; + TStringBuf Value; + + private: + enum EBit { + FEATURE_NAME(Filter), + FEATURE_NAME(SortByName), + FEATURE_NAME(RemoveEmptyQuery), + FEATURE_NAME(RewriteDirty), + _FeatureMAX + }; + + public: + enum EPublic : ui32 { + FeatureMAX = _FeatureMAX, + FEATURE_FLAG(Filter), + FEATURE_FLAG(SortByName), + FEATURE_FLAG(RemoveEmptyQuery), + FEATURE_FLAG(RewriteDirty), + }; + + enum EProcessed { + // OK and clean. + ProcessedOK = 0, + + // OK, but query stored in internal buffer and TUri::Rewrite() is required. + ProcessedDirty = 1, + + ProcessedMalformed = 2, + ProcessedTooMany = 3, + }; + }; + + typedef bool (*TQueryArgFilter)(const TQueryArg& arg, void* filterData); #undef FEATURE_NAME #undef FEATURE_FLAG_NAME #undef FEATURE_FLAG - const char* FieldToString(const TField::EField& t); - const char* ParsedStateToString(const TState::EParsed& t); - const char* SchemeKindToString(const TScheme::EKind& t); + const char* FieldToString(const TField::EField& t); + const char* ParsedStateToString(const TState::EParsed& t); + const char* SchemeKindToString(const TScheme::EKind& t); } -Y_DECLARE_OUT_SPEC(inline, NUri::TField::EField, out, t) { +Y_DECLARE_OUT_SPEC(inline, NUri::TField::EField, out, t) { out << NUri::FieldToString(t); } -Y_DECLARE_OUT_SPEC(inline, NUri::TScheme::EKind, out, t) { +Y_DECLARE_OUT_SPEC(inline, NUri::TScheme::EKind, out, t) { out << NUri::SchemeKindToString(t); } -Y_DECLARE_OUT_SPEC(inline, NUri::TState::EParsed, out, t) { +Y_DECLARE_OUT_SPEC(inline, NUri::TState::EParsed, out, t) { out << NUri::ParsedStateToString(t); } -static inline ui16 DefaultPort(NUri::TScheme::EKind scheme) { +static inline ui16 DefaultPort(NUri::TScheme::EKind scheme) { return NUri::TSchemeInfo::GetDefaultPort(scheme); } -static inline NUri::TScheme::EKind SchemeKind(const TStringBuf& scheme) { +static inline NUri::TScheme::EKind SchemeKind(const TStringBuf& scheme) { return NUri::TSchemeInfo::GetKind(scheme); } diff --git a/library/cpp/uri/encode.cpp b/library/cpp/uri/encode.cpp index 584fb1bac9..34790c2d21 100644 --- a/library/cpp/uri/encode.cpp +++ b/library/cpp/uri/encode.cpp @@ -1,9 +1,9 @@ #include "encode.h" -#include <util/generic/singleton.h> +#include <util/generic/singleton.h> namespace NUri { - namespace NEncode { + namespace NEncode { // http://tools.ietf.org/html/rfc3986#section-2.2 #define GENDELIMS0 ":/?#[]@" #define SUBDELIMS0 "!$&'()*+,;=" @@ -24,197 +24,197 @@ namespace NUri { // rest allowed in qry, frag #define SUBDELIMS2 "!$'()*," - const TEncoder::TGrammar& TEncoder::Grammar() { - return *Singleton<TEncoder::TGrammar>(); - } - - // initialize the grammar map - TEncoder::TGrammar::TGrammar() { - // first set up unreserved characters safe in any field - const ui64 featUnres = TFeature::FeatureDecodeUnreserved; - AddRng('0', '9', ECFDigit, featUnres); - AddRng('A', 'Z', ECFUpper, featUnres | TFeature::FeatureToLower); - AddRng('a', 'z', ECFLower, featUnres); - Add(UNRESERVED, ECFUnres, featUnres); - - // XXX: standard "safe" set used previously "-_.!~*();/:@$,", with comment: - // alnum + reserved + mark + ( '[', ']') - ('=' '+' '&' '\'' '"' '\\' '?') - Add("!*();/:@$,", ECFStdrd, TFeature::FeatureDecodeStandardExtra); - - // now field-specific subsets of reserved characters (gen-delims + sub-delims) - const ui64 featSafe = TFeature::FeatureDecodeFieldAllowed; - - Add(GENDELIMS1, 0, featSafe, TField::FlagPath | TField::FlagQuery | TField::FlagFrag); - Add(GENDELIMS2, 0, featSafe, TField::FlagQuery | TField::FlagFrag); - - Add(SUBDELIMS1, 0, featSafe, TField::FlagUser); - Add(SUBDELIMS2, 0, featSafe, TField::FlagUser | TField::FlagQuery | TField::FlagFrag); - - // control chars - AddRng(0x00, 0x20, TFeature::FeatureEncodeCntrl); - Add(0x7f, TFeature::FeatureEncodeCntrl); - - // '%' starts a percent-encoded sequence - Add('%', TFeature::FeatureDecodeANY | TFeature::FeatureEncodePercent); - - // extended ASCII - AddRng(128, 255, TFeature::FeatureEncodeExtendedASCII | TFeature::FeatureDecodeExtendedASCII); - - // extended delims - Add("\"<>[\\]^`{|}", TFeature::FeatureEncodeExtendedDelim | TFeature::FeatureDecodeExtendedDelim); - - // add characters with other features - Add(' ', TFeature::FeatureEncodeSpace | TFeature::FeatureEncodeSpaceAsPlus); - Add("'\"\\", TFeature::FeatureEncodeForSQL); - - GetMutable(':').EncodeFld |= TField::FlagUser; - GetMutable('?').EncodeFld |= TField::FlagPath; - GetMutable('#').EncodeFld |= TField::FlagPath | TField::FlagQuery; - GetMutable('&').EncodeFld |= TField::FlagQuery; - GetMutable('+').EncodeFld |= TField::FlagQuery; - } - - // should we decode an encoded character - bool TCharFlags::IsDecode(ui32 fldmask, ui64 flags) const { - const ui64 myflags = flags & FeatFlags; - if (myflags & TFeature::FeaturesEncode) - return false; - if (myflags & TFeature::FeaturesDecode) - return true; - return (fldmask & DecodeFld) && (flags & TFeature::FeatureDecodeFieldAllowed); - } - - const int dD = 'a' - 'A'; - - int TEncodeMapper::EncodeSym(unsigned char& ch) const { - const TCharFlags& chflags = TEncoder::GetFlags(ch); - const ui64 flags = Flags & chflags.FeatFlags; - - if (flags & TFeature::FeatureToLower) - ch += dD; - - if (Q_DecodeAny) - return -1; - - if (flags & TFeature::FeaturesEncode) - return 1; - - if (' ' == ch) { - if (Q_EncodeSpcAsPlus) - ch = '+'; - return 0; - } - - return 0; - } - - int TEncodeMapper::EncodeHex(unsigned char& ch) const { - const TCharFlags& chflags = TEncoder::GetFlags(ch); - const ui64 flags = Flags & chflags.FeatFlags; - - if (flags & TFeature::FeatureToLower) - ch += dD; - - if (Q_DecodeAny) - return -1; - - if (chflags.IsDecode(FldMask, Flags)) - return 0; - - if (' ' == ch) { - if (!Q_EncodeSpcAsPlus) - return 1; - ch = '+'; - return 0; - } - + const TEncoder::TGrammar& TEncoder::Grammar() { + return *Singleton<TEncoder::TGrammar>(); + } + + // initialize the grammar map + TEncoder::TGrammar::TGrammar() { + // first set up unreserved characters safe in any field + const ui64 featUnres = TFeature::FeatureDecodeUnreserved; + AddRng('0', '9', ECFDigit, featUnres); + AddRng('A', 'Z', ECFUpper, featUnres | TFeature::FeatureToLower); + AddRng('a', 'z', ECFLower, featUnres); + Add(UNRESERVED, ECFUnres, featUnres); + + // XXX: standard "safe" set used previously "-_.!~*();/:@$,", with comment: + // alnum + reserved + mark + ( '[', ']') - ('=' '+' '&' '\'' '"' '\\' '?') + Add("!*();/:@$,", ECFStdrd, TFeature::FeatureDecodeStandardExtra); + + // now field-specific subsets of reserved characters (gen-delims + sub-delims) + const ui64 featSafe = TFeature::FeatureDecodeFieldAllowed; + + Add(GENDELIMS1, 0, featSafe, TField::FlagPath | TField::FlagQuery | TField::FlagFrag); + Add(GENDELIMS2, 0, featSafe, TField::FlagQuery | TField::FlagFrag); + + Add(SUBDELIMS1, 0, featSafe, TField::FlagUser); + Add(SUBDELIMS2, 0, featSafe, TField::FlagUser | TField::FlagQuery | TField::FlagFrag); + + // control chars + AddRng(0x00, 0x20, TFeature::FeatureEncodeCntrl); + Add(0x7f, TFeature::FeatureEncodeCntrl); + + // '%' starts a percent-encoded sequence + Add('%', TFeature::FeatureDecodeANY | TFeature::FeatureEncodePercent); + + // extended ASCII + AddRng(128, 255, TFeature::FeatureEncodeExtendedASCII | TFeature::FeatureDecodeExtendedASCII); + + // extended delims + Add("\"<>[\\]^`{|}", TFeature::FeatureEncodeExtendedDelim | TFeature::FeatureDecodeExtendedDelim); + + // add characters with other features + Add(' ', TFeature::FeatureEncodeSpace | TFeature::FeatureEncodeSpaceAsPlus); + Add("'\"\\", TFeature::FeatureEncodeForSQL); + + GetMutable(':').EncodeFld |= TField::FlagUser; + GetMutable('?').EncodeFld |= TField::FlagPath; + GetMutable('#').EncodeFld |= TField::FlagPath | TField::FlagQuery; + GetMutable('&').EncodeFld |= TField::FlagQuery; + GetMutable('+').EncodeFld |= TField::FlagQuery; + } + + // should we decode an encoded character + bool TCharFlags::IsDecode(ui32 fldmask, ui64 flags) const { + const ui64 myflags = flags & FeatFlags; + if (myflags & TFeature::FeaturesEncode) + return false; + if (myflags & TFeature::FeaturesDecode) + return true; + return (fldmask & DecodeFld) && (flags & TFeature::FeatureDecodeFieldAllowed); + } + + const int dD = 'a' - 'A'; + + int TEncodeMapper::EncodeSym(unsigned char& ch) const { + const TCharFlags& chflags = TEncoder::GetFlags(ch); + const ui64 flags = Flags & chflags.FeatFlags; + + if (flags & TFeature::FeatureToLower) + ch += dD; + + if (Q_DecodeAny) + return -1; + + if (flags & TFeature::FeaturesEncode) + return 1; + + if (' ' == ch) { + if (Q_EncodeSpcAsPlus) + ch = '+'; + return 0; + } + + return 0; + } + + int TEncodeMapper::EncodeHex(unsigned char& ch) const { + const TCharFlags& chflags = TEncoder::GetFlags(ch); + const ui64 flags = Flags & chflags.FeatFlags; + + if (flags & TFeature::FeatureToLower) + ch += dD; + + if (Q_DecodeAny) + return -1; + + if (chflags.IsDecode(FldMask, Flags)) + return 0; + + if (' ' == ch) { + if (!Q_EncodeSpcAsPlus) + return 1; + ch = '+'; + return 0; + } + return 1; - } - - bool TEncodeToMapper::Encode(unsigned char ch) const { - if (Q_DecodeAny) - return false; - - const TCharFlags& chflags = TEncoder::GetFlags(ch); - if (FldMask & chflags.EncodeFld) - return true; - - const ui64 flags = Flags & chflags.FeatFlags; - return (flags & TFeature::FeaturesEncode); - } - - TEncoder::TEncoder(IOutputStream& out, const TEncodeMapper& fldsrc, const TEncodeToMapper& flddst) - : Out(out) - , FldSrc(fldsrc) - , FldDst(flddst) - , OutFlags(0) - , HexValue(0) - { - } - - IOutputStream& TEncoder::Hex(IOutputStream& out, unsigned char val) { - static const char sHexCodes[] = "0123456789ABCDEF"; - return out << sHexCodes[(val >> 4) & 0xF] << sHexCodes[val & 0xF]; - } - - IOutputStream& TEncoder::EncodeAll(IOutputStream& out, const TStringBuf& val) { - for (size_t i = 0; i != val.length(); ++i) - Encode(out, val[i]); - return out; - } - - IOutputStream& TEncoder::EncodeNotAlnum(IOutputStream& out, const TStringBuf& val) { - for (size_t i = 0; i != val.length(); ++i) { - const char c = val[i]; - if (IsAlnum(c)) - out << c; - else - Encode(out, c); - } - return out; - } - - IOutputStream& TEncoder::EncodeField( - IOutputStream& out, const TStringBuf& val, TField::EField fld) { - const ui32 fldmask = ui32(1) << fld; - for (size_t i = 0; i != val.length(); ++i) { - const char ch = val[i]; - if (GetFlags(ch).IsAllowed(fldmask)) - out << ch; - else - Encode(out, ch); - } - return out; - } - - IOutputStream& TEncoder::EncodeField( - IOutputStream& out, const TStringBuf& val, TField::EField fld, ui64 flags) { - const ui32 fldmask = ui32(1) << fld; - for (size_t i = 0; i != val.length(); ++i) { - const char ch = val[i]; - if (GetFlags(ch).IsDecode(fldmask, flags)) - out << ch; - else - Encode(out, ch); - } - return out; - } - - void TEncoder::Do(unsigned char ch, int res) { - OutFlags |= GetFlags(ch).FeatFlags; - - bool escapepct = false; - if (0 < res) // definitely encode - escapepct = FldDst.Enabled(); - else if (0 != res || !FldDst.Enabled() || !FldDst.Encode(ch)) { - Out << ch; - return; - } - - Out << '%'; - if (escapepct) - Out.Write("25", 2); // '%' - Hex(Out, ch); - } + } + + bool TEncodeToMapper::Encode(unsigned char ch) const { + if (Q_DecodeAny) + return false; + + const TCharFlags& chflags = TEncoder::GetFlags(ch); + if (FldMask & chflags.EncodeFld) + return true; + + const ui64 flags = Flags & chflags.FeatFlags; + return (flags & TFeature::FeaturesEncode); + } + + TEncoder::TEncoder(IOutputStream& out, const TEncodeMapper& fldsrc, const TEncodeToMapper& flddst) + : Out(out) + , FldSrc(fldsrc) + , FldDst(flddst) + , OutFlags(0) + , HexValue(0) + { + } + + IOutputStream& TEncoder::Hex(IOutputStream& out, unsigned char val) { + static const char sHexCodes[] = "0123456789ABCDEF"; + return out << sHexCodes[(val >> 4) & 0xF] << sHexCodes[val & 0xF]; + } + + IOutputStream& TEncoder::EncodeAll(IOutputStream& out, const TStringBuf& val) { + for (size_t i = 0; i != val.length(); ++i) + Encode(out, val[i]); + return out; + } + + IOutputStream& TEncoder::EncodeNotAlnum(IOutputStream& out, const TStringBuf& val) { + for (size_t i = 0; i != val.length(); ++i) { + const char c = val[i]; + if (IsAlnum(c)) + out << c; + else + Encode(out, c); + } + return out; + } + + IOutputStream& TEncoder::EncodeField( + IOutputStream& out, const TStringBuf& val, TField::EField fld) { + const ui32 fldmask = ui32(1) << fld; + for (size_t i = 0; i != val.length(); ++i) { + const char ch = val[i]; + if (GetFlags(ch).IsAllowed(fldmask)) + out << ch; + else + Encode(out, ch); + } + return out; + } + + IOutputStream& TEncoder::EncodeField( + IOutputStream& out, const TStringBuf& val, TField::EField fld, ui64 flags) { + const ui32 fldmask = ui32(1) << fld; + for (size_t i = 0; i != val.length(); ++i) { + const char ch = val[i]; + if (GetFlags(ch).IsDecode(fldmask, flags)) + out << ch; + else + Encode(out, ch); + } + return out; + } + + void TEncoder::Do(unsigned char ch, int res) { + OutFlags |= GetFlags(ch).FeatFlags; + + bool escapepct = false; + if (0 < res) // definitely encode + escapepct = FldDst.Enabled(); + else if (0 != res || !FldDst.Enabled() || !FldDst.Encode(ch)) { + Out << ch; + return; + } + + Out << '%'; + if (escapepct) + Out.Write("25", 2); // '%' + Hex(Out, ch); + } } } diff --git a/library/cpp/uri/encode.h b/library/cpp/uri/encode.h index a9ece15427..ed1f1309ce 100644 --- a/library/cpp/uri/encode.h +++ b/library/cpp/uri/encode.h @@ -5,278 +5,278 @@ #include <util/stream/output.h> namespace NUri { - namespace NEncode { -#define CHAR_TYPE_NAME(f) _ECT##f -#define CHAR_TYPE_FLAG(f) ECF##f = 1u << CHAR_TYPE_NAME(f) - - enum ECharType { - CHAR_TYPE_NAME(Digit), - CHAR_TYPE_NAME(Lower), - CHAR_TYPE_NAME(Upper), - CHAR_TYPE_NAME(Unres), - CHAR_TYPE_NAME(Stdrd), - }; - - enum ECharFlag { - CHAR_TYPE_FLAG(Digit), - CHAR_TYPE_FLAG(Lower), - CHAR_TYPE_FLAG(Upper), - CHAR_TYPE_FLAG(Unres), - CHAR_TYPE_FLAG(Stdrd), - // compound group flags - ECGAlpha = ECFUpper | ECFLower, - ECGAlnum = ECGAlpha | ECFDigit, - ECGUnres = ECGAlnum | ECFUnres, - ECGStdrd = ECGUnres | ECFStdrd, - }; + namespace NEncode { +#define CHAR_TYPE_NAME(f) _ECT##f +#define CHAR_TYPE_FLAG(f) ECF##f = 1u << CHAR_TYPE_NAME(f) + + enum ECharType { + CHAR_TYPE_NAME(Digit), + CHAR_TYPE_NAME(Lower), + CHAR_TYPE_NAME(Upper), + CHAR_TYPE_NAME(Unres), + CHAR_TYPE_NAME(Stdrd), + }; + + enum ECharFlag { + CHAR_TYPE_FLAG(Digit), + CHAR_TYPE_FLAG(Lower), + CHAR_TYPE_FLAG(Upper), + CHAR_TYPE_FLAG(Unres), + CHAR_TYPE_FLAG(Stdrd), + // compound group flags + ECGAlpha = ECFUpper | ECFLower, + ECGAlnum = ECGAlpha | ECFDigit, + ECGUnres = ECGAlnum | ECFUnres, + ECGStdrd = ECGUnres | ECFStdrd, + }; #undef CHAR_TYPE_NAME #undef CHAR_TYPE_FLAG - struct TCharFlags { - ui32 TypeFlags; - ui64 FeatFlags; - ui32 DecodeFld; // decode if FeatureDecodeFieldAllowed - ui32 EncodeFld; // encode if shouldn't be treated as delimiter - TCharFlags(ui64 feat = 0) - : TypeFlags(0) - , FeatFlags(feat) - , DecodeFld(0) - , EncodeFld(0) - { - } - TCharFlags(ui32 type, ui64 feat, ui32 decmask = 0, ui32 encmask = 0) - : TypeFlags(type) - , FeatFlags(feat) - , DecodeFld(decmask) - , EncodeFld(encmask) - { - } - TCharFlags& Add(const TCharFlags& val) { - TypeFlags |= val.TypeFlags; - FeatFlags |= val.FeatFlags; - DecodeFld |= val.DecodeFld; - EncodeFld |= val.EncodeFld; - return *this; - } - bool IsAllowed(ui32 fldmask) const { - return (TypeFlags & ECGUnres) || (DecodeFld & ~EncodeFld & fldmask); - } - // should we decode an encoded character - bool IsDecode(ui32 fldmask, ui64 flags) const; - }; - - class TEncodeMapperBase { - protected: - TEncodeMapperBase() - : Flags(0) - , FldMask(0) - , Q_DecodeAny(false) - { - } - TEncodeMapperBase(ui64 flags, TField::EField fld) - : Flags(flags) - , FldMask(1u << fld) - , Q_DecodeAny(flags & TFeature::FeatureDecodeANY) - { - } - - protected: - const ui64 Flags; - const ui32 FldMask; - const bool Q_DecodeAny; // this is a special option for username/password - }; - - // maps a sym or hex character and indicates whether it has to be encoded - class TEncodeMapper - : public TEncodeMapperBase { - public: - TEncodeMapper(ui64 flags, TField::EField fld = TField::FieldAllMAX) - : TEncodeMapperBase(flags, fld) - , Q_EncodeSpcAsPlus(flags & TFeature::FeatureEncodeSpaceAsPlus) - { - } - // negative=sym, positive=hex, zero=maybesym - int EncodeSym(unsigned char&) const; - int EncodeHex(unsigned char&) const; - - protected: - const bool Q_EncodeSpcAsPlus; - }; - - // indicates whether a character has to be encoded when copying to a field - class TEncodeToMapper - : public TEncodeMapperBase { - public: - TEncodeToMapper() - : TEncodeMapperBase() - { - } - TEncodeToMapper(ui64 flags, TField::EField fld = TField::FieldAllMAX) - : TEncodeMapperBase(flags, fld) - { - } - bool Enabled() const { - return 0 != FldMask; - } - bool Encode(unsigned char) const; - }; - - class TEncoder { - public: - TEncoder(IOutputStream& out, const TEncodeMapper& fldsrc, const TEncodeToMapper& flddst = TEncodeToMapper()); - - ui64 ReEncode(const TStringBuf& url); - ui64 ReEncode(const char* str, size_t len) { - return ReEncode(TStringBuf(str, len)); - } - - protected: - static bool IsType(unsigned char c, ui64 flags) { - return GetFlags(c).TypeFlags & flags; - } - - public: - static bool IsDigit(unsigned char c) { - return IsType(c, ECFDigit); - } - static bool IsUpper(unsigned char c) { - return IsType(c, ECFUpper); - } - static bool IsLower(unsigned char c) { - return IsType(c, ECFLower); - } - static bool IsAlpha(unsigned char c) { - return IsType(c, ECGAlpha); - } - static bool IsAlnum(unsigned char c) { - return IsType(c, ECGAlnum); - } - static bool IsUnres(unsigned char c) { - return IsType(c, ECGUnres); - } - static const TCharFlags& GetFlags(unsigned char c) { - return Grammar().Get(c); - } - - public: - // process an encoded string, decoding safe chars and encoding unsafe - static IOutputStream& ReEncode(IOutputStream& out, const TStringBuf& val, const TEncodeMapper& srcfld) { - TEncoder(out, srcfld).ReEncode(val); - return out; - } - static IOutputStream& ReEncodeTo(IOutputStream& out, const TStringBuf& val, const TEncodeMapper& srcfld, const TEncodeToMapper& dstfld) { - TEncoder(out, srcfld, dstfld).ReEncode(val); - return out; - } - - // see also UrlUnescape() from string/quote.h - static IOutputStream& Decode( - IOutputStream& out, const TStringBuf& val, ui64 flags) { - return ReEncode(out, val, flags | TFeature::FeatureDecodeANY); - } - - public: - // process a raw string or char, encode as needed - static IOutputStream& Hex(IOutputStream& out, unsigned char val); - static IOutputStream& Encode(IOutputStream& out, unsigned char val) { - out << '%'; - return Hex(out, val); - } - static IOutputStream& EncodeAll(IOutputStream& out, const TStringBuf& val); - static IOutputStream& EncodeNotAlnum(IOutputStream& out, const TStringBuf& val); - - static IOutputStream& EncodeField(IOutputStream& out, const TStringBuf& val, TField::EField fld); - static IOutputStream& EncodeField(IOutputStream& out, const TStringBuf& val, TField::EField fld, ui64 flags); - - static IOutputStream& Encode(IOutputStream& out, const TStringBuf& val) { - return EncodeField(out, val, TField::FieldAllMAX); - } - - static IOutputStream& Encode(IOutputStream& out, const TStringBuf& val, ui64 flags) { - return EncodeField(out, val, TField::FieldAllMAX, flags); - } - - public: - class TGrammar { - TCharFlags Map_[256]; - - public: - TGrammar(); - const TCharFlags& Get(unsigned char ch) const { - return Map_[ch]; - } - - TCharFlags& GetMutable(unsigned char ch) { - return Map_[ch]; - } - TCharFlags& Add(unsigned char ch, const TCharFlags& val) { - return GetMutable(ch).Add(val); - } - - void AddRng(unsigned char lo, unsigned char hi, const TCharFlags& val) { - for (unsigned i = lo; i <= hi; ++i) - Add(i, val); - } - void AddRng(unsigned char lo, unsigned char hi, ui32 type, ui64 feat, ui32 decmask = 0, ui32 encmask = 0) { - AddRng(lo, hi, TCharFlags(type, feat, decmask, encmask)); - } - - void Add(const TStringBuf& set, const TCharFlags& val) { - for (size_t i = 0; i != set.length(); ++i) - Add(set[i], val); - } - void Add(const TStringBuf& set, ui32 type, ui64 feat, ui32 decmask = 0, ui32 encmask = 0) { - Add(set, TCharFlags(type, feat, decmask, encmask)); - } - }; - - static const TGrammar& Grammar(); - - protected: - IOutputStream& Out; - const TEncodeMapper FldSrc; - const TEncodeToMapper FldDst; - ui64 OutFlags; - int HexValue; - - protected: - void HexReset() { - HexValue = 0; - } - - void HexDigit(char c) { - HexAdd(c - '0'); - } - void HexUpper(char c) { - HexAdd(c - 'A' + 10); - } - void HexLower(char c) { - HexAdd(c - 'a' + 10); - } - - void HexAdd(int val) { - HexValue <<= 4; - HexValue += val; - } - - protected: - void DoSym(unsigned char ch) { - const int res = FldSrc.EncodeSym(ch); - Do(ch, res); - } - void DoHex(unsigned char ch) { - const int res = FldSrc.EncodeHex(ch); - Do(ch, res); - } - void DoHex() { - DoHex(HexValue); - HexValue = 0; - } - void Do(unsigned char, int); - }; + struct TCharFlags { + ui32 TypeFlags; + ui64 FeatFlags; + ui32 DecodeFld; // decode if FeatureDecodeFieldAllowed + ui32 EncodeFld; // encode if shouldn't be treated as delimiter + TCharFlags(ui64 feat = 0) + : TypeFlags(0) + , FeatFlags(feat) + , DecodeFld(0) + , EncodeFld(0) + { + } + TCharFlags(ui32 type, ui64 feat, ui32 decmask = 0, ui32 encmask = 0) + : TypeFlags(type) + , FeatFlags(feat) + , DecodeFld(decmask) + , EncodeFld(encmask) + { + } + TCharFlags& Add(const TCharFlags& val) { + TypeFlags |= val.TypeFlags; + FeatFlags |= val.FeatFlags; + DecodeFld |= val.DecodeFld; + EncodeFld |= val.EncodeFld; + return *this; + } + bool IsAllowed(ui32 fldmask) const { + return (TypeFlags & ECGUnres) || (DecodeFld & ~EncodeFld & fldmask); + } + // should we decode an encoded character + bool IsDecode(ui32 fldmask, ui64 flags) const; + }; + + class TEncodeMapperBase { + protected: + TEncodeMapperBase() + : Flags(0) + , FldMask(0) + , Q_DecodeAny(false) + { + } + TEncodeMapperBase(ui64 flags, TField::EField fld) + : Flags(flags) + , FldMask(1u << fld) + , Q_DecodeAny(flags & TFeature::FeatureDecodeANY) + { + } + + protected: + const ui64 Flags; + const ui32 FldMask; + const bool Q_DecodeAny; // this is a special option for username/password + }; + + // maps a sym or hex character and indicates whether it has to be encoded + class TEncodeMapper + : public TEncodeMapperBase { + public: + TEncodeMapper(ui64 flags, TField::EField fld = TField::FieldAllMAX) + : TEncodeMapperBase(flags, fld) + , Q_EncodeSpcAsPlus(flags & TFeature::FeatureEncodeSpaceAsPlus) + { + } + // negative=sym, positive=hex, zero=maybesym + int EncodeSym(unsigned char&) const; + int EncodeHex(unsigned char&) const; + + protected: + const bool Q_EncodeSpcAsPlus; + }; + + // indicates whether a character has to be encoded when copying to a field + class TEncodeToMapper + : public TEncodeMapperBase { + public: + TEncodeToMapper() + : TEncodeMapperBase() + { + } + TEncodeToMapper(ui64 flags, TField::EField fld = TField::FieldAllMAX) + : TEncodeMapperBase(flags, fld) + { + } + bool Enabled() const { + return 0 != FldMask; + } + bool Encode(unsigned char) const; + }; + + class TEncoder { + public: + TEncoder(IOutputStream& out, const TEncodeMapper& fldsrc, const TEncodeToMapper& flddst = TEncodeToMapper()); + + ui64 ReEncode(const TStringBuf& url); + ui64 ReEncode(const char* str, size_t len) { + return ReEncode(TStringBuf(str, len)); + } + + protected: + static bool IsType(unsigned char c, ui64 flags) { + return GetFlags(c).TypeFlags & flags; + } + + public: + static bool IsDigit(unsigned char c) { + return IsType(c, ECFDigit); + } + static bool IsUpper(unsigned char c) { + return IsType(c, ECFUpper); + } + static bool IsLower(unsigned char c) { + return IsType(c, ECFLower); + } + static bool IsAlpha(unsigned char c) { + return IsType(c, ECGAlpha); + } + static bool IsAlnum(unsigned char c) { + return IsType(c, ECGAlnum); + } + static bool IsUnres(unsigned char c) { + return IsType(c, ECGUnres); + } + static const TCharFlags& GetFlags(unsigned char c) { + return Grammar().Get(c); + } + + public: + // process an encoded string, decoding safe chars and encoding unsafe + static IOutputStream& ReEncode(IOutputStream& out, const TStringBuf& val, const TEncodeMapper& srcfld) { + TEncoder(out, srcfld).ReEncode(val); + return out; + } + static IOutputStream& ReEncodeTo(IOutputStream& out, const TStringBuf& val, const TEncodeMapper& srcfld, const TEncodeToMapper& dstfld) { + TEncoder(out, srcfld, dstfld).ReEncode(val); + return out; + } + + // see also UrlUnescape() from string/quote.h + static IOutputStream& Decode( + IOutputStream& out, const TStringBuf& val, ui64 flags) { + return ReEncode(out, val, flags | TFeature::FeatureDecodeANY); + } + + public: + // process a raw string or char, encode as needed + static IOutputStream& Hex(IOutputStream& out, unsigned char val); + static IOutputStream& Encode(IOutputStream& out, unsigned char val) { + out << '%'; + return Hex(out, val); + } + static IOutputStream& EncodeAll(IOutputStream& out, const TStringBuf& val); + static IOutputStream& EncodeNotAlnum(IOutputStream& out, const TStringBuf& val); + + static IOutputStream& EncodeField(IOutputStream& out, const TStringBuf& val, TField::EField fld); + static IOutputStream& EncodeField(IOutputStream& out, const TStringBuf& val, TField::EField fld, ui64 flags); + + static IOutputStream& Encode(IOutputStream& out, const TStringBuf& val) { + return EncodeField(out, val, TField::FieldAllMAX); + } + + static IOutputStream& Encode(IOutputStream& out, const TStringBuf& val, ui64 flags) { + return EncodeField(out, val, TField::FieldAllMAX, flags); + } + + public: + class TGrammar { + TCharFlags Map_[256]; + + public: + TGrammar(); + const TCharFlags& Get(unsigned char ch) const { + return Map_[ch]; + } + + TCharFlags& GetMutable(unsigned char ch) { + return Map_[ch]; + } + TCharFlags& Add(unsigned char ch, const TCharFlags& val) { + return GetMutable(ch).Add(val); + } + + void AddRng(unsigned char lo, unsigned char hi, const TCharFlags& val) { + for (unsigned i = lo; i <= hi; ++i) + Add(i, val); + } + void AddRng(unsigned char lo, unsigned char hi, ui32 type, ui64 feat, ui32 decmask = 0, ui32 encmask = 0) { + AddRng(lo, hi, TCharFlags(type, feat, decmask, encmask)); + } + + void Add(const TStringBuf& set, const TCharFlags& val) { + for (size_t i = 0; i != set.length(); ++i) + Add(set[i], val); + } + void Add(const TStringBuf& set, ui32 type, ui64 feat, ui32 decmask = 0, ui32 encmask = 0) { + Add(set, TCharFlags(type, feat, decmask, encmask)); + } + }; + + static const TGrammar& Grammar(); + + protected: + IOutputStream& Out; + const TEncodeMapper FldSrc; + const TEncodeToMapper FldDst; + ui64 OutFlags; + int HexValue; + + protected: + void HexReset() { + HexValue = 0; + } + + void HexDigit(char c) { + HexAdd(c - '0'); + } + void HexUpper(char c) { + HexAdd(c - 'A' + 10); + } + void HexLower(char c) { + HexAdd(c - 'a' + 10); + } + + void HexAdd(int val) { + HexValue <<= 4; + HexValue += val; + } + + protected: + void DoSym(unsigned char ch) { + const int res = FldSrc.EncodeSym(ch); + Do(ch, res); + } + void DoHex(unsigned char ch) { + const int res = FldSrc.EncodeHex(ch); + Do(ch, res); + } + void DoHex() { + DoHex(HexValue); + HexValue = 0; + } + void Do(unsigned char, int); + }; } - using TEncoder = NEncode::TEncoder; + using TEncoder = NEncode::TEncoder; } diff --git a/library/cpp/uri/http_url.h b/library/cpp/uri/http_url.h index 7c8e8d844d..92a332a1f6 100644 --- a/library/cpp/uri/http_url.h +++ b/library/cpp/uri/http_url.h @@ -2,11 +2,11 @@ #include "uri.h" #include "other.h" - + // XXX: use NUri::TUri directly; this whole file is for backwards compatibility class THttpURL - : public NUri::TUri { + : public NUri::TUri { public: typedef TField::EFlags TFlags; typedef TField::EField TField; @@ -16,42 +16,42 @@ public: public: enum { FeatureUnescapeStandard = TFeature::FeatureDecodeStandard, - FeatureEscSpace = TFeature::FeatureEncodeSpaceAsPlus, - FeatureEscapeUnescaped = TFeature::FeatureEncodeExtendedASCII, - FeatureNormalPath = TFeature::FeaturePathStripRootParent, + FeatureEscSpace = TFeature::FeatureEncodeSpaceAsPlus, + FeatureEscapeUnescaped = TFeature::FeatureEncodeExtendedASCII, + FeatureNormalPath = TFeature::FeaturePathStripRootParent, }; public: THttpURL(unsigned defaultPort = 80) : TUri(defaultPort) - { - } + { + } - THttpURL(const TStringBuf& host, ui16 port, const TStringBuf& path, const TStringBuf& query = TStringBuf(), const TStringBuf& scheme = "http", unsigned defaultPort = 0) + THttpURL(const TStringBuf& host, ui16 port, const TStringBuf& path, const TStringBuf& query = TStringBuf(), const TStringBuf& scheme = "http", unsigned defaultPort = 0) : TUri(host, port, path, query, scheme, defaultPort) - { - } + { + } THttpURL(const TUri& url) : TUri(url) - { - } + { + } public: // XXX: don't use any of these legacy methods below public: // use TUri::GetField() instead /// will return null-terminated if fld is not dirty - const char* Get(EField fld) const { + const char* Get(EField fld) const { return GetField(fld).data(); } public: // use TUriUpdate class so that Rewrite() is only called once - void Set(EField field, const TStringBuf& value) { + void Set(EField field, const TStringBuf& value) { if (SetInMemory(field, value)) Rewrite(); } template <size_t size> - void Set(EField field, const char (&value)[size]) { + void Set(EField field, const char (&value)[size]) { if (SetInMemory(field, value)) Rewrite(); } @@ -59,19 +59,19 @@ public: // use TUriUpdate class so that Rewrite() is only called once public: // use TUri::FldXXX methods for better control // Partial quick set of the field, can be called for // multiple fields - bool SetInMemory(EField field, const TStringBuf& value) { + bool SetInMemory(EField field, const TStringBuf& value) { return FldMemSet(field, value); } // clears a field - void Reset(EField field) { + void Reset(EField field) { FldClr(field); } }; -static inline const char* HttpURLParsedStateToString(const NUri::TState::EParsed& t) { +static inline const char* HttpURLParsedStateToString(const NUri::TState::EParsed& t) { return NUri::ParsedStateToString(t); } -static inline const char* HttpUrlSchemeKindToString(const NUri::TScheme::EKind& t) { +static inline const char* HttpUrlSchemeKindToString(const NUri::TScheme::EKind& t) { return NUri::SchemeKindToString(t); } diff --git a/library/cpp/uri/location.cpp b/library/cpp/uri/location.cpp index a6a4d11ffa..2eb099beda 100644 --- a/library/cpp/uri/location.cpp +++ b/library/cpp/uri/location.cpp @@ -2,30 +2,30 @@ #include "uri.h" namespace NUri { - static const int URI_PARSE_FLAGS = - (TFeature::FeaturesRecommended | TFeature::FeatureConvertHostIDN | TFeature::FeatureEncodeExtendedDelim | TFeature::FeatureEncodePercent) & ~TFeature::FeatureHashBangToEscapedFragment; + static const int URI_PARSE_FLAGS = + (TFeature::FeaturesRecommended | TFeature::FeatureConvertHostIDN | TFeature::FeatureEncodeExtendedDelim | TFeature::FeatureEncodePercent) & ~TFeature::FeatureHashBangToEscapedFragment; - TString ResolveRedirectLocation(const TStringBuf& baseUrl, - const TStringBuf& location) { - TUri baseUri; - TUri locationUri; + TString ResolveRedirectLocation(const TStringBuf& baseUrl, + const TStringBuf& location) { + TUri baseUri; + TUri locationUri; - // Parse base URL. - if (baseUri.Parse(baseUrl, URI_PARSE_FLAGS) != NUri::TState::ParsedOK) { - return ""; - } - // Parse location with respect to the base URL. - if (locationUri.Parse(location, baseUri, URI_PARSE_FLAGS) != NUri::TState::ParsedOK) { - return ""; - } - // Inherit fragment. - if (!locationUri.GetField(NUri::TField::FieldFragment)) { - NUri::TUriUpdate update(locationUri); - update.Set(NUri::TField::FieldFragment, baseUri.GetField(NUri::TField::FieldFragment)); - } - TString res; - locationUri.Print(res, NUri::TField::FlagAllFields); - return res; + // Parse base URL. + if (baseUri.Parse(baseUrl, URI_PARSE_FLAGS) != NUri::TState::ParsedOK) { + return ""; + } + // Parse location with respect to the base URL. + if (locationUri.Parse(location, baseUri, URI_PARSE_FLAGS) != NUri::TState::ParsedOK) { + return ""; + } + // Inherit fragment. + if (!locationUri.GetField(NUri::TField::FieldFragment)) { + NUri::TUriUpdate update(locationUri); + update.Set(NUri::TField::FieldFragment, baseUri.GetField(NUri::TField::FieldFragment)); + } + TString res; + locationUri.Print(res, NUri::TField::FlagAllFields); + return res; } } diff --git a/library/cpp/uri/location.h b/library/cpp/uri/location.h index 0f533fe0b5..9478510fd3 100644 --- a/library/cpp/uri/location.h +++ b/library/cpp/uri/location.h @@ -3,11 +3,11 @@ #include <util/generic/string.h> namespace NUri { - /** + /** * Resolve Location header according to https://tools.ietf.org/html/rfc7231#section-7.1.2 * * @return Resolved location's url or empty string in case of any error. */ - TString ResolveRedirectLocation(const TStringBuf& baseUrl, const TStringBuf& location); + TString ResolveRedirectLocation(const TStringBuf& baseUrl, const TStringBuf& location); } diff --git a/library/cpp/uri/location_ut.cpp b/library/cpp/uri/location_ut.cpp index 26a0f64471..e3140fffff 100644 --- a/library/cpp/uri/location_ut.cpp +++ b/library/cpp/uri/location_ut.cpp @@ -5,36 +5,36 @@ Y_UNIT_TEST_SUITE(TResolveRedirectTests) { Y_UNIT_TEST(Absolute) { UNIT_ASSERT_EQUAL( - NUri::ResolveRedirectLocation("http://example.com", "http://redir-example.com/sub"), "http://redir-example.com/sub"); + NUri::ResolveRedirectLocation("http://example.com", "http://redir-example.com/sub"), "http://redir-example.com/sub"); } Y_UNIT_TEST(AbsWithFragment) { UNIT_ASSERT_EQUAL( - NUri::ResolveRedirectLocation("http://example.com", "http://redir-example.com/sub#Hello"), "http://redir-example.com/sub#Hello"); + NUri::ResolveRedirectLocation("http://example.com", "http://redir-example.com/sub#Hello"), "http://redir-example.com/sub#Hello"); UNIT_ASSERT_EQUAL( - NUri::ResolveRedirectLocation("http://example.com/#Hello", "http://redir-example.com/sub"), "http://redir-example.com/sub#Hello"); + NUri::ResolveRedirectLocation("http://example.com/#Hello", "http://redir-example.com/sub"), "http://redir-example.com/sub#Hello"); } Y_UNIT_TEST(Rel) { UNIT_ASSERT_EQUAL( - NUri::ResolveRedirectLocation("http://example.com", "/sub"), "http://example.com/sub"); + NUri::ResolveRedirectLocation("http://example.com", "/sub"), "http://example.com/sub"); } Y_UNIT_TEST(RelWithFragment) { UNIT_ASSERT_EQUAL( - NUri::ResolveRedirectLocation("http://example.com", "/sub#Hello"), "http://example.com/sub#Hello"); + NUri::ResolveRedirectLocation("http://example.com", "/sub#Hello"), "http://example.com/sub#Hello"); UNIT_ASSERT_EQUAL( - NUri::ResolveRedirectLocation("http://example.com/#Hello", "/sub"), "http://example.com/sub#Hello"); + NUri::ResolveRedirectLocation("http://example.com/#Hello", "/sub"), "http://example.com/sub#Hello"); } Y_UNIT_TEST(WrongLocation) { UNIT_ASSERT_EQUAL( - NUri::ResolveRedirectLocation("http://example.com", ""), ""); + NUri::ResolveRedirectLocation("http://example.com", ""), ""); } Y_UNIT_TEST(WrongBase) { UNIT_ASSERT_EQUAL( - NUri::ResolveRedirectLocation("", "http://example.com"), ""); + NUri::ResolveRedirectLocation("", "http://example.com"), ""); } Y_UNIT_TEST(HashBangIsNothingSpecial) { UNIT_ASSERT_EQUAL( - NUri::ResolveRedirectLocation("http://example.com", "http://redir-example.com/sub#!Hello"), "http://redir-example.com/sub#!Hello"); + NUri::ResolveRedirectLocation("http://example.com", "http://redir-example.com/sub#!Hello"), "http://redir-example.com/sub#!Hello"); UNIT_ASSERT_EQUAL( - NUri::ResolveRedirectLocation("http://example.com/#!Hello", "http://redir-example.com/sub"), "http://redir-example.com/sub#!Hello"); + NUri::ResolveRedirectLocation("http://example.com/#!Hello", "http://redir-example.com/sub"), "http://redir-example.com/sub#!Hello"); } } diff --git a/library/cpp/uri/other.cpp b/library/cpp/uri/other.cpp index b23a5b68a9..7e02d70e25 100644 --- a/library/cpp/uri/other.cpp +++ b/library/cpp/uri/other.cpp @@ -1,35 +1,35 @@ #include "other.h" - + #include <util/string/util.h> #include <util/system/yassert.h> - + /********************************************************/ /********************************************************/ static const Tr InvertTr(".:/?#", "\005\004\003\002\001"); static const Tr RevertTr("\005\004\003\002\001", ".:/?#"); -void TrspChars(char* s) { +void TrspChars(char* s) { InvertTr.Do(s); } -void UnTrspChars(char* s) { +void UnTrspChars(char* s) { RevertTr.Do(s); } -void TrspChars(char* s, size_t l) { +void TrspChars(char* s, size_t l) { InvertTr.Do(s, l); } -void UnTrspChars(char* s, size_t l) { +void UnTrspChars(char* s, size_t l) { RevertTr.Do(s, l); } -void TrspChars(const char* s, char* d) { +void TrspChars(const char* s, char* d) { InvertTr.Do(s, d); } -void UnTrspChars(const char* s, char* d) { +void UnTrspChars(const char* s, char* d) { RevertTr.Do(s, d); } @@ -57,14 +57,14 @@ void InvertDomain(char* begin, char* end) { e = end; for (size_t i = 0, n = e - b; i < n / 2; ++i) DoSwap(b[i], b[n - i - 1]); - if (e == end) + if (e == end) break; b = e + 1; } } void InvertUrl(char* begin, char* end) { - char* slash = strchr(begin, '/'); + char* slash = strchr(begin, '/'); if (slash) { *slash = 0; } diff --git a/library/cpp/uri/other.h b/library/cpp/uri/other.h index 7aec22e77b..621d89a32f 100644 --- a/library/cpp/uri/other.h +++ b/library/cpp/uri/other.h @@ -5,12 +5,12 @@ // Some functions for inverted url representation // No scheme cut-off, no 80th port normalization -void TrspChars(char* s); -void UnTrspChars(char* s); -void TrspChars(char* s, size_t l); -void UnTrspChars(char* s, size_t l); -void TrspChars(const char* s, char* d); -void UnTrspChars(const char* s, char* d); +void TrspChars(char* s); +void UnTrspChars(char* s); +void TrspChars(char* s, size_t l); +void UnTrspChars(char* s, size_t l); +void TrspChars(const char* s, char* d); +void UnTrspChars(const char* s, char* d); void InvertDomain(char* begin, char* end); diff --git a/library/cpp/uri/parse.cpp b/library/cpp/uri/parse.cpp index 1db4e008c4..76d8a9fee3 100644 --- a/library/cpp/uri/parse.cpp +++ b/library/cpp/uri/parse.cpp @@ -1,207 +1,207 @@ -#include "parse.h" +#include "parse.h" #include "common.h" #include "encode.h" namespace NUri { - const TParseFlags TParser::FieldFlags[] = - { - TParseFlags(0 // FieldScheme - | TFeature::FeatureToLower, - 0) - - , - TParseFlags(0 // FieldUsername - | TFeature::FeatureDecodeANY | TFeature::FeaturesDecode | TFeature::FeatureEncodePercent, - 0 | TFeature::FeatureToLower) - - , - TParseFlags(0 // FieldPassword - | TFeature::FeatureDecodeANY | TFeature::FeaturesDecode | TFeature::FeatureEncodePercent, - 0 | TFeature::FeatureToLower) - - , - TParseFlags(0 // FieldHost - | TFeature::FeatureToLower | TFeature::FeatureUpperEncoded | (TFeature::FeaturesMaybeEncode & ~TFeature::FeatureEncodeExtendedDelim), - 0 | TFeature::FeaturesMaybeDecode) - - , - TParseFlags(0 // FieldPort - , - 0) - - , - TParseFlags(0 // FieldPath - | TFeature::FeaturesEncodePChar | TFeature::FeaturePathOperation, - 0 | TFeature::FeatureToLower | TFeature::FeatureEncodeSpaceAsPlus) - - , - TParseFlags(0 // FieldQuery - | TFeature::FeaturesEncodePChar | TFeature::FeatureEncodeSpaceAsPlus, - 0 | TFeature::FeatureToLower) - - , - TParseFlags(0 // FieldFragment - | TFeature::FeaturesEncodePChar, - 0 | TFeature::FeatureToLower | TFeature::FeatureEncodeSpaceAsPlus)}; - - namespace NParse { - void TRange::AddRange(const TRange& range, ui64 mask) { - FlagsAllPlaintext |= range.FlagsAllPlaintext; - // update only if flags apply here - mask &= range.FlagsEncodeMasked; - if (0 == mask) - return; - FlagsEncodeMasked |= mask; - if (mask & TFeature::FeaturesMaybeEncode) - Encode += range.Encode; - if (mask & TFeature::FeaturesDecode) - Decode += range.Decode; - } - - } - - void TParser::copyRequirementsImpl(const char* ptr) { - Y_ASSERT(0 != CurRange.FlagsAllPlaintext); - Y_UNUSED(ptr); + const TParseFlags TParser::FieldFlags[] = + { + TParseFlags(0 // FieldScheme + | TFeature::FeatureToLower, + 0) + + , + TParseFlags(0 // FieldUsername + | TFeature::FeatureDecodeANY | TFeature::FeaturesDecode | TFeature::FeatureEncodePercent, + 0 | TFeature::FeatureToLower) + + , + TParseFlags(0 // FieldPassword + | TFeature::FeatureDecodeANY | TFeature::FeaturesDecode | TFeature::FeatureEncodePercent, + 0 | TFeature::FeatureToLower) + + , + TParseFlags(0 // FieldHost + | TFeature::FeatureToLower | TFeature::FeatureUpperEncoded | (TFeature::FeaturesMaybeEncode & ~TFeature::FeatureEncodeExtendedDelim), + 0 | TFeature::FeaturesMaybeDecode) + + , + TParseFlags(0 // FieldPort + , + 0) + + , + TParseFlags(0 // FieldPath + | TFeature::FeaturesEncodePChar | TFeature::FeaturePathOperation, + 0 | TFeature::FeatureToLower | TFeature::FeatureEncodeSpaceAsPlus) + + , + TParseFlags(0 // FieldQuery + | TFeature::FeaturesEncodePChar | TFeature::FeatureEncodeSpaceAsPlus, + 0 | TFeature::FeatureToLower) + + , + TParseFlags(0 // FieldFragment + | TFeature::FeaturesEncodePChar, + 0 | TFeature::FeatureToLower | TFeature::FeatureEncodeSpaceAsPlus)}; + + namespace NParse { + void TRange::AddRange(const TRange& range, ui64 mask) { + FlagsAllPlaintext |= range.FlagsAllPlaintext; + // update only if flags apply here + mask &= range.FlagsEncodeMasked; + if (0 == mask) + return; + FlagsEncodeMasked |= mask; + if (mask & TFeature::FeaturesMaybeEncode) + Encode += range.Encode; + if (mask & TFeature::FeaturesDecode) + Decode += range.Decode; + } + + } + + void TParser::copyRequirementsImpl(const char* ptr) { + Y_ASSERT(0 != CurRange.FlagsAllPlaintext); + Y_UNUSED(ptr); #ifdef DO_PRN - PrintHead(ptr, __FUNCTION__) + PrintHead(ptr, __FUNCTION__) << " all=[" << IntToString<16>(CurRange.FlagsAllPlaintext) << "] enc=[" << IntToString<16>(CurRange.FlagsEncodeMasked) - << " & " << IntToString<16>(Flags.Allow | Flags.Extra) << "]"; - PrintTail(CurRange.Beg, ptr); + << " & " << IntToString<16>(Flags.Allow | Flags.Extra) << "]"; + PrintTail(CurRange.Beg, ptr); #endif - for (int i = 0; i < TField::FieldUrlMAX; ++i) { - const TField::EField fld = TField::EField(i); - TSection& section = Sections[fld]; - // update only sections in progress - if (nullptr == section.Beg) - continue; - // and overlapping with the range - if (nullptr != section.End && section.End < CurRange.Beg) - continue; -#ifdef DO_PRN - PrintHead(ptr, __FUNCTION__, fld) - << " all=[" << IntToString<16>(CurRange.FlagsAllPlaintext) - << "] enc=[" << IntToString<16>(CurRange.FlagsEncodeMasked) - << " & " << IntToString<16>(GetFieldFlags(fld)) << "]"; - PrintTail(section.Beg, ptr); -#endif - section.AddRange(CurRange, GetFieldFlags(fld)); - } - CurRange.Reset(); + for (int i = 0; i < TField::FieldUrlMAX; ++i) { + const TField::EField fld = TField::EField(i); + TSection& section = Sections[fld]; + // update only sections in progress + if (nullptr == section.Beg) + continue; + // and overlapping with the range + if (nullptr != section.End && section.End < CurRange.Beg) + continue; +#ifdef DO_PRN + PrintHead(ptr, __FUNCTION__, fld) + << " all=[" << IntToString<16>(CurRange.FlagsAllPlaintext) + << "] enc=[" << IntToString<16>(CurRange.FlagsEncodeMasked) + << " & " << IntToString<16>(GetFieldFlags(fld)) << "]"; + PrintTail(section.Beg, ptr); +#endif + section.AddRange(CurRange, GetFieldFlags(fld)); + } + CurRange.Reset(); } - void TParser::PctEndImpl(const char* ptr) { + void TParser::PctEndImpl(const char* ptr) { #ifdef DO_PRN - PrintHead(PctBegin, __FUNCTION__); - PrintTail(PctBegin, ptr); + PrintHead(PctBegin, __FUNCTION__); + PrintTail(PctBegin, ptr); #else - Y_UNUSED(ptr); + Y_UNUSED(ptr); #endif - setRequirement(PctBegin, TEncoder::GetFlags('%').FeatFlags); - PctBegin = nullptr; - } + setRequirement(PctBegin, TEncoder::GetFlags('%').FeatFlags); + PctBegin = nullptr; + } - void TParser::HexSet(const char* ptr) { - Y_ASSERT(nullptr != PctBegin); + void TParser::HexSet(const char* ptr) { + Y_ASSERT(nullptr != PctBegin); #ifdef DO_PRN - PrintHead(ptr, __FUNCTION__); - PrintTail(PctBegin, ptr + 1); + PrintHead(ptr, __FUNCTION__); + PrintTail(PctBegin, ptr + 1); #endif - PctBegin = nullptr; - const unsigned char ch = HexValue; - ui64 flags = TEncoder::GetFlags('%').FeatFlags | TEncoder::GetFlags(ch).FeatFlags; + PctBegin = nullptr; + const unsigned char ch = HexValue; + ui64 flags = TEncoder::GetFlags('%').FeatFlags | TEncoder::GetFlags(ch).FeatFlags; - setRequirementExcept(ptr, flags, TFeature::FeaturesMaybeEncode); - } + setRequirementExcept(ptr, flags, TFeature::FeaturesMaybeEncode); + } - TState::EParsed TParser::ParseImpl() { + TState::EParsed TParser::ParseImpl() { #ifdef DO_PRN - PrintHead(UriStr.data(), "[Parsing]") << "URL"; - PrintTail(UriStr); + PrintHead(UriStr.data(), "[Parsing]") << "URL"; + PrintTail(UriStr); #endif - const bool ok = doParse(UriStr.data(), UriStr.length()); + const bool ok = doParse(UriStr.data(), UriStr.length()); #ifdef DO_PRN - Cdbg << (ok ? "[Parsed]" : "[Failed]"); - for (int idx = 0; idx < TField::FieldUrlMAX; ++idx) { - const TSection& section = Sections[idx]; - if (section.IsSet()) - Cdbg << ' ' << TField::EField(idx) << "=[" << section.Get() << ']'; - } - Cdbg << Endl; + Cdbg << (ok ? "[Parsed]" : "[Failed]"); + for (int idx = 0; idx < TField::FieldUrlMAX; ++idx) { + const TSection& section = Sections[idx]; + if (section.IsSet()) + Cdbg << ' ' << TField::EField(idx) << "=[" << section.Get() << ']'; + } + Cdbg << Endl; #endif - if (!ok) { - if (!(Flags & TFeature::FeatureTryToFix) || !Sections[TField::FieldFrag].Beg) - return TState::ParsedBadFormat; - //Here: error was in fragment, just ignore it - ResetSection(TField::FieldFrag); - } - - if ((Flags & TFeature::FeatureDenyNetworkPath) && IsNetPath()) + if (!ok) { + if (!(Flags & TFeature::FeatureTryToFix) || !Sections[TField::FieldFrag].Beg) + return TState::ParsedBadFormat; + //Here: error was in fragment, just ignore it + ResetSection(TField::FieldFrag); + } + + if ((Flags & TFeature::FeatureDenyNetworkPath) && IsNetPath()) return TState::ParsedBadFormat; - const TSection& scheme = Sections[TField::FieldScheme]; - Scheme = scheme.IsSet() ? TSchemeInfo::GetKind(scheme.Get()) : TScheme::SchemeEmpty; - const TSchemeInfo& schemeInfo = TSchemeInfo::Get(Scheme); + const TSection& scheme = Sections[TField::FieldScheme]; + Scheme = scheme.IsSet() ? TSchemeInfo::GetKind(scheme.Get()) : TScheme::SchemeEmpty; + const TSchemeInfo& schemeInfo = TSchemeInfo::Get(Scheme); - if (IsRootless()) { - // opaque case happens - if (schemeInfo.FldReq & TField::FlagHost) - return TState::ParsedBadFormat; + if (IsRootless()) { + // opaque case happens + if (schemeInfo.FldReq & TField::FlagHost) + return TState::ParsedBadFormat; - if (TScheme::SchemeEmpty == Scheme) - return TState::ParsedBadScheme; + if (TScheme::SchemeEmpty == Scheme) + return TState::ParsedBadScheme; - if (Flags & TFeature::FeatureAllowRootless) - return TState::ParsedOK; + if (Flags & TFeature::FeatureAllowRootless) + return TState::ParsedOK; - if (!(Flags & TFeature::FeatureSchemeFlexible)) - return TState::ParsedBadScheme; + if (!(Flags & TFeature::FeatureSchemeFlexible)) + return TState::ParsedBadScheme; - return TState::ParsedRootless; - } + return TState::ParsedRootless; + } + + checkSectionCollision(TField::FieldUser, TField::FieldHost); + checkSectionCollision(TField::FieldPass, TField::FieldPort); + + if (0 == (Flags & TFeature::FeatureAuthSupported)) + if (Sections[TField::FieldUser].IsSet() || Sections[TField::FieldPass].IsSet()) + return TState::ParsedBadAuth; + + TSection& host = Sections[TField::FieldHost]; + if (host.IsSet()) + for (; host.End != host.Beg && '.' == host.End[-1];) + --host.End; - checkSectionCollision(TField::FieldUser, TField::FieldHost); - checkSectionCollision(TField::FieldPass, TField::FieldPort); - - if (0 == (Flags & TFeature::FeatureAuthSupported)) - if (Sections[TField::FieldUser].IsSet() || Sections[TField::FieldPass].IsSet()) - return TState::ParsedBadAuth; - - TSection& host = Sections[TField::FieldHost]; - if (host.IsSet()) - for (; host.End != host.Beg && '.' == host.End[-1];) - --host.End; - - if (scheme.IsSet()) { - ui64 wantCareFlags = 0; - switch (Scheme) { - case TScheme::SchemeHTTP: - break; - case TScheme::SchemeEmpty: - Scheme = TScheme::SchemeUnknown; + if (scheme.IsSet()) { + ui64 wantCareFlags = 0; + switch (Scheme) { + case TScheme::SchemeHTTP: + break; + case TScheme::SchemeEmpty: + Scheme = TScheme::SchemeUnknown; [[fallthrough]]; - case TScheme::SchemeUnknown: - wantCareFlags = - TFeature::FeatureSchemeFlexible | TFeature::FeatureNoRelPath; - break; - default: - wantCareFlags = - TFeature::FeatureSchemeFlexible | TFeature::FeatureSchemeKnown; - break; - } - - if (0 != wantCareFlags && 0 == (Flags & wantCareFlags)) - return TState::ParsedBadScheme; - if ((schemeInfo.FldReq & TField::FlagHost) || (Flags & TFeature::FeatureRemoteOnly)) - if (!host.IsSet() || 0 == host.Len()) - return TState::ParsedBadFormat; + case TScheme::SchemeUnknown: + wantCareFlags = + TFeature::FeatureSchemeFlexible | TFeature::FeatureNoRelPath; + break; + default: + wantCareFlags = + TFeature::FeatureSchemeFlexible | TFeature::FeatureSchemeKnown; + break; + } + + if (0 != wantCareFlags && 0 == (Flags & wantCareFlags)) + return TState::ParsedBadScheme; + if ((schemeInfo.FldReq & TField::FlagHost) || (Flags & TFeature::FeatureRemoteOnly)) + if (!host.IsSet() || 0 == host.Len()) + return TState::ParsedBadFormat; } - return TState::ParsedOK; + return TState::ParsedOK; } } diff --git a/library/cpp/uri/parse.h b/library/cpp/uri/parse.h index ca2358e572..f4b56ba1c4 100644 --- a/library/cpp/uri/parse.h +++ b/library/cpp/uri/parse.h @@ -13,349 +13,349 @@ #include <util/system/yassert.h> namespace NUri { - class TParser; - - namespace NParse { - class TRange { - public: - const char* Beg; - ui64 FlagsEncodeMasked; - ui64 FlagsAllPlaintext; - ui32 Encode; - ui32 Decode; - - public: - TRange(const char* beg = nullptr) - : Beg(beg) - , FlagsEncodeMasked(0) - , FlagsAllPlaintext(0) - , Encode(0) - , Decode(0) - { - } - - void Reset(const char* beg = nullptr) { - *this = TRange(beg); - } - - void AddRange(const TRange& range, ui64 mask); - - void AddFlag(const char* ptr, ui64 mask, ui64 flag) { - if (0 != flag) - AddFlagImpl(ptr, mask, flag, flag); - } - - void AddFlagExcept(const char* ptr, ui64 mask, ui64 flag, ui64 exclflag) { - if (0 != flag) - AddFlagImpl(ptr, mask, flag & ~exclflag, flag); - } - - void AddFlagUnless(const char* ptr, ui64 mask, ui64 flag, ui64 exclmask) { - if (0 != flag) - AddFlagImpl(ptr, mask, flag, flag, exclmask); - } - - void AddFlag(const char* ptr, ui64 mask, ui64 flag, ui64 exclflag, ui64 exclmask) { - if (0 != flag) - AddFlagImpl(ptr, mask, flag & ~exclflag, flag, exclmask); - } - - private: - void AddFlagImpl(const char* ptr, ui64 mask, ui64 plainflag, ui64 encflag) { - AddFlagAllPlaintextImpl(ptr, plainflag); - AddFlagEncodeMaskedImpl(encflag & mask); - } - - void AddFlagImpl(const char* ptr, ui64 mask, ui64 plainflag, ui64 encflag, ui64 exclmask) { - AddFlagAllPlaintextImpl(ptr, plainflag); - if (0 == (mask & exclmask)) - AddFlagEncodeMaskedImpl(encflag & mask); - } - - void AddFlagAllPlaintextImpl(const char* ptr, ui64 flag) { - if (nullptr == Beg) - Beg = ptr; - FlagsAllPlaintext |= flag; - } - - void AddFlagEncodeMaskedImpl(ui64 flag) { - if (0 == flag) - return; - FlagsEncodeMasked |= flag; - if (flag & TFeature::FeaturesMaybeEncode) - ++Encode; - else if (flag & TFeature::FeaturesDecode) - ++Decode; - } - }; + class TParser; + + namespace NParse { + class TRange { + public: + const char* Beg; + ui64 FlagsEncodeMasked; + ui64 FlagsAllPlaintext; + ui32 Encode; + ui32 Decode; + + public: + TRange(const char* beg = nullptr) + : Beg(beg) + , FlagsEncodeMasked(0) + , FlagsAllPlaintext(0) + , Encode(0) + , Decode(0) + { + } + + void Reset(const char* beg = nullptr) { + *this = TRange(beg); + } + + void AddRange(const TRange& range, ui64 mask); + + void AddFlag(const char* ptr, ui64 mask, ui64 flag) { + if (0 != flag) + AddFlagImpl(ptr, mask, flag, flag); + } + + void AddFlagExcept(const char* ptr, ui64 mask, ui64 flag, ui64 exclflag) { + if (0 != flag) + AddFlagImpl(ptr, mask, flag & ~exclflag, flag); + } + + void AddFlagUnless(const char* ptr, ui64 mask, ui64 flag, ui64 exclmask) { + if (0 != flag) + AddFlagImpl(ptr, mask, flag, flag, exclmask); + } + + void AddFlag(const char* ptr, ui64 mask, ui64 flag, ui64 exclflag, ui64 exclmask) { + if (0 != flag) + AddFlagImpl(ptr, mask, flag & ~exclflag, flag, exclmask); + } + + private: + void AddFlagImpl(const char* ptr, ui64 mask, ui64 plainflag, ui64 encflag) { + AddFlagAllPlaintextImpl(ptr, plainflag); + AddFlagEncodeMaskedImpl(encflag & mask); + } + + void AddFlagImpl(const char* ptr, ui64 mask, ui64 plainflag, ui64 encflag, ui64 exclmask) { + AddFlagAllPlaintextImpl(ptr, plainflag); + if (0 == (mask & exclmask)) + AddFlagEncodeMaskedImpl(encflag & mask); + } + + void AddFlagAllPlaintextImpl(const char* ptr, ui64 flag) { + if (nullptr == Beg) + Beg = ptr; + FlagsAllPlaintext |= flag; + } + + void AddFlagEncodeMaskedImpl(ui64 flag) { + if (0 == flag) + return; + FlagsEncodeMasked |= flag; + if (flag & TFeature::FeaturesMaybeEncode) + ++Encode; + else if (flag & TFeature::FeaturesDecode) + ++Decode; + } + }; } - class TSection - : protected NParse::TRange { - private: - friend class TParser; - - private: - const char* End; - - TSection(const char* beg = nullptr) - : NParse::TRange(beg) - , End(nullptr) - { - } - - void Reset() { - Enter(nullptr); - } - - void Reset(const char* pc) { - Y_ASSERT(!Beg || !pc || Beg < pc); - Reset(); - } - - void Enter(const char* pc) { - *this = TSection(pc); - } - - bool Leave(const char* pc) { - Y_ASSERT(Beg); - End = pc; - return true; - } - - void Set(const TStringBuf& buf) { - Enter(buf.data()); - Leave(buf.data() + buf.length()); - } - - public: - bool IsSet() const { - return End; - } - - TStringBuf Get() const { - return TStringBuf(Beg, End); - } - - size_t Len() const { - return End - Beg; - } - - size_t DecodedLen() const { - return Len() - 2 * Decode; - } - - size_t EncodedLen() const { - return 2 * Encode + DecodedLen(); - } - - ui32 GetEncode() const { - return Encode; - } - - ui32 GetDecode() const { - return Decode; - } - - ui64 GetFlagsEncode() const { - return FlagsEncodeMasked; - } - - ui64 GetFlagsAllPlaintext() const { - return FlagsAllPlaintext; - } - }; - - class TParser { - public: - TSection Sections[TField::FieldUrlMAX]; - TScheme::EKind Scheme; - const TParseFlags Flags; - const TStringBuf UriStr; - TState::EParsed State; - ECharset Enc; - - public: - TParser(const TParseFlags& flags, const TStringBuf& uri, ECharset enc = CODES_UTF8) - : Scheme(TScheme::SchemeEmpty) - , Flags(flags | TFeature::FeatureDecodeANY) - , UriStr(uri) - , State(TState::ParsedEmpty) - , Enc(enc) - , HexValue(0) - , PctBegin(nullptr) - { - Y_ASSERT(0 == (Flags & TFeature::FeaturePathOperation) - // can't define all of them - || TFeature::FeaturesPath != (Flags & TFeature::FeaturesPath)); - State = ParseImpl(); - } - - public: - const TSection& Get(TField::EField fld) const { - return Sections[fld]; - } - TSection& GetMutable(TField::EField fld) { - return Sections[fld]; - } - bool Has(TField::EField fld) const { - return Get(fld).IsSet(); - } - bool IsNetPath() const { - return Has(TField::FieldHost) && 2 < UriStr.length() && '/' == UriStr[0] && '/' == UriStr[1]; - } - bool IsRootless() const { - return Has(TField::FieldScheme) && !Has(TField::FieldHost) && (!Has(TField::FieldPath) || '/' != Get(TField::FieldPath).Get()[0]); - } - // for RFC 2396 compatibility - bool IsOpaque() const { - return IsRootless(); - } - static ui64 GetFieldFlags(TField::EField fld, const TParseFlags& flags) { - return FieldFlags[fld] & flags; - } - ui64 GetFieldFlags(TField::EField fld) const { - return GetFieldFlags(fld, Flags); - } - - protected: - static const TParseFlags FieldFlags[TField::FieldUrlMAX]; - TSection::TRange CurRange; - unsigned HexValue; - const char* PctBegin; + class TSection + : protected NParse::TRange { + private: + friend class TParser; + + private: + const char* End; + + TSection(const char* beg = nullptr) + : NParse::TRange(beg) + , End(nullptr) + { + } + + void Reset() { + Enter(nullptr); + } + + void Reset(const char* pc) { + Y_ASSERT(!Beg || !pc || Beg < pc); + Reset(); + } + + void Enter(const char* pc) { + *this = TSection(pc); + } + + bool Leave(const char* pc) { + Y_ASSERT(Beg); + End = pc; + return true; + } + + void Set(const TStringBuf& buf) { + Enter(buf.data()); + Leave(buf.data() + buf.length()); + } + + public: + bool IsSet() const { + return End; + } + + TStringBuf Get() const { + return TStringBuf(Beg, End); + } + + size_t Len() const { + return End - Beg; + } + + size_t DecodedLen() const { + return Len() - 2 * Decode; + } + + size_t EncodedLen() const { + return 2 * Encode + DecodedLen(); + } + + ui32 GetEncode() const { + return Encode; + } + + ui32 GetDecode() const { + return Decode; + } + + ui64 GetFlagsEncode() const { + return FlagsEncodeMasked; + } + + ui64 GetFlagsAllPlaintext() const { + return FlagsAllPlaintext; + } + }; + + class TParser { + public: + TSection Sections[TField::FieldUrlMAX]; + TScheme::EKind Scheme; + const TParseFlags Flags; + const TStringBuf UriStr; + TState::EParsed State; + ECharset Enc; + + public: + TParser(const TParseFlags& flags, const TStringBuf& uri, ECharset enc = CODES_UTF8) + : Scheme(TScheme::SchemeEmpty) + , Flags(flags | TFeature::FeatureDecodeANY) + , UriStr(uri) + , State(TState::ParsedEmpty) + , Enc(enc) + , HexValue(0) + , PctBegin(nullptr) + { + Y_ASSERT(0 == (Flags & TFeature::FeaturePathOperation) + // can't define all of them + || TFeature::FeaturesPath != (Flags & TFeature::FeaturesPath)); + State = ParseImpl(); + } + + public: + const TSection& Get(TField::EField fld) const { + return Sections[fld]; + } + TSection& GetMutable(TField::EField fld) { + return Sections[fld]; + } + bool Has(TField::EField fld) const { + return Get(fld).IsSet(); + } + bool IsNetPath() const { + return Has(TField::FieldHost) && 2 < UriStr.length() && '/' == UriStr[0] && '/' == UriStr[1]; + } + bool IsRootless() const { + return Has(TField::FieldScheme) && !Has(TField::FieldHost) && (!Has(TField::FieldPath) || '/' != Get(TField::FieldPath).Get()[0]); + } + // for RFC 2396 compatibility + bool IsOpaque() const { + return IsRootless(); + } + static ui64 GetFieldFlags(TField::EField fld, const TParseFlags& flags) { + return FieldFlags[fld] & flags; + } + ui64 GetFieldFlags(TField::EField fld) const { + return GetFieldFlags(fld, Flags); + } + + protected: + static const TParseFlags FieldFlags[TField::FieldUrlMAX]; + TSection::TRange CurRange; + unsigned HexValue; + const char* PctBegin; #ifdef DO_PRN - IOutputStream& PrintAddr(const char* ptr) const { - return Cdbg << "[" << IntToString<16>(ui64(ptr)) << "] "; - } - - IOutputStream& PrintHead(const char* ptr, const char* func) const { - return PrintAddr(ptr) << func << " "; - } - - IOutputStream& PrintHead(const char* ptr, const char* func, const TField::EField& fld) const { - return PrintHead(ptr, func) << fld; - } - - IOutputStream& PrintTail(const TStringBuf& val) const { - return Cdbg << " [" << val << "]" << Endl; - } - IOutputStream& PrintTail(const char* beg, const char* end) const { - return PrintTail(TStringBuf(beg, end)); - } + IOutputStream& PrintAddr(const char* ptr) const { + return Cdbg << "[" << IntToString<16>(ui64(ptr)) << "] "; + } + + IOutputStream& PrintHead(const char* ptr, const char* func) const { + return PrintAddr(ptr) << func << " "; + } + + IOutputStream& PrintHead(const char* ptr, const char* func, const TField::EField& fld) const { + return PrintHead(ptr, func) << fld; + } + + IOutputStream& PrintTail(const TStringBuf& val) const { + return Cdbg << " [" << val << "]" << Endl; + } + IOutputStream& PrintTail(const char* beg, const char* end) const { + return PrintTail(TStringBuf(beg, end)); + } #endif - void ResetSection(TField::EField fld, const char* pc = nullptr) { + void ResetSection(TField::EField fld, const char* pc = nullptr) { #ifdef DO_PRN - PrintHead(pc, __FUNCTION__, fld); - PrintTail(pc); + PrintHead(pc, __FUNCTION__, fld); + PrintTail(pc); #endif - Sections[fld].Reset(pc); - } + Sections[fld].Reset(pc); + } - void storeSection(const TStringBuf& val, TField::EField fld) { + void storeSection(const TStringBuf& val, TField::EField fld) { #ifdef DO_PRN - PrintHead(val.data(), __FUNCTION__, fld); - PrintTail(val); + PrintHead(val.data(), __FUNCTION__, fld); + PrintTail(val); #endif - Sections[fld].Set(val); - } + Sections[fld].Set(val); + } - void startSection(const char* pc, TField::EField fld) { + void startSection(const char* pc, TField::EField fld) { #ifdef DO_PRN - PrintHead(pc, __FUNCTION__, fld); - PrintTail(pc); + PrintHead(pc, __FUNCTION__, fld); + PrintTail(pc); #endif - copyRequirements(pc); - Sections[fld].Enter(pc); - } + copyRequirements(pc); + Sections[fld].Enter(pc); + } - void finishSection(const char* pc, TField::EField fld) { + void finishSection(const char* pc, TField::EField fld) { #ifdef DO_PRN - PrintHead(pc, __FUNCTION__, fld); - PrintTail(pc); + PrintHead(pc, __FUNCTION__, fld); + PrintTail(pc); #endif - if (Sections[fld].Leave(pc)) - copyRequirements(pc); - } + if (Sections[fld].Leave(pc)) + copyRequirements(pc); + } - void setRequirement(const char* ptr, ui64 flags) { + void setRequirement(const char* ptr, ui64 flags) { #ifdef DO_PRN - PrintHead(ptr, __FUNCTION__) << IntToString<16>(flags) - << " & mask=" << IntToString<16>(Flags.Allow | Flags.Extra); - PrintTail(ptr); + PrintHead(ptr, __FUNCTION__) << IntToString<16>(flags) + << " & mask=" << IntToString<16>(Flags.Allow | Flags.Extra); + PrintTail(ptr); #endif - CurRange.AddFlag(ptr, Flags.Allow | Flags.Extra, flags); - } + CurRange.AddFlag(ptr, Flags.Allow | Flags.Extra, flags); + } - void setRequirementExcept(const char* ptr, ui64 flags, ui64 exclflag) { + void setRequirementExcept(const char* ptr, ui64 flags, ui64 exclflag) { #ifdef DO_PRN - PrintHead(ptr, __FUNCTION__) << IntToString<16>(flags) - << " & exclflag=" << IntToString<16>(exclflag) - << " & mask=" << IntToString<16>(Flags.Allow | Flags.Extra); - PrintTail(ptr); + PrintHead(ptr, __FUNCTION__) << IntToString<16>(flags) + << " & exclflag=" << IntToString<16>(exclflag) + << " & mask=" << IntToString<16>(Flags.Allow | Flags.Extra); + PrintTail(ptr); #endif - CurRange.AddFlagExcept(ptr, Flags.Allow | Flags.Extra, flags, exclflag); - } + CurRange.AddFlagExcept(ptr, Flags.Allow | Flags.Extra, flags, exclflag); + } - void setRequirementUnless(const char* ptr, ui64 flags, ui64 exclmask) { + void setRequirementUnless(const char* ptr, ui64 flags, ui64 exclmask) { #ifdef DO_PRN - PrintHead(ptr, __FUNCTION__) << IntToString<16>(flags) - << " & exclmask=" << IntToString<16>(exclmask) - << " & mask=" << IntToString<16>(Flags.Allow | Flags.Extra); - PrintTail(ptr); + PrintHead(ptr, __FUNCTION__) << IntToString<16>(flags) + << " & exclmask=" << IntToString<16>(exclmask) + << " & mask=" << IntToString<16>(Flags.Allow | Flags.Extra); + PrintTail(ptr); #endif - CurRange.AddFlagUnless(ptr, Flags.Allow | Flags.Extra, flags, exclmask); - } - - void copyRequirementsImpl(const char* ptr); - void copyRequirements(const char* ptr) { - PctEnd(ptr); - if (nullptr != CurRange.Beg && CurRange.Beg != ptr) - copyRequirementsImpl(ptr); - } - - void HexDigit(const char* ptr, char c) { - Y_UNUSED(ptr); - HexAdd(c - '0'); - } - void HexUpper(const char* ptr, char c) { - setRequirementUnless(ptr, TFeature::FeatureToLower, TFeature::FeatureUpperEncoded); - HexAdd(c - 'A' + 10); - } - void HexLower(const char* ptr, char c) { - setRequirement(ptr, TFeature::FeatureUpperEncoded); - HexAdd(c - 'a' + 10); - } - void HexAdd(unsigned val) { - HexValue <<= 4; - HexValue += val; - } - void HexReset() { - HexValue = 0; - } - void HexSet(const char* ptr); - - void PctEndImpl(const char* ptr); - void PctEnd(const char* ptr) { - if (nullptr != PctBegin && ptr != PctBegin) - PctEndImpl(ptr); - } - void PctBeg(const char* ptr) { - PctEnd(ptr); - HexReset(); - PctBegin = ptr; - } - - void checkSectionCollision(TField::EField fld1, TField::EField fld2) { - if (Sections[fld1].IsSet() && Sections[fld2].IsSet() && Sections[fld1].Beg == Sections[fld2].Beg) { - Sections[fld1].Reset(); - } - } - - bool doParse(const char* str_beg, size_t length); - TState::EParsed ParseImpl(); - }; + CurRange.AddFlagUnless(ptr, Flags.Allow | Flags.Extra, flags, exclmask); + } + + void copyRequirementsImpl(const char* ptr); + void copyRequirements(const char* ptr) { + PctEnd(ptr); + if (nullptr != CurRange.Beg && CurRange.Beg != ptr) + copyRequirementsImpl(ptr); + } + + void HexDigit(const char* ptr, char c) { + Y_UNUSED(ptr); + HexAdd(c - '0'); + } + void HexUpper(const char* ptr, char c) { + setRequirementUnless(ptr, TFeature::FeatureToLower, TFeature::FeatureUpperEncoded); + HexAdd(c - 'A' + 10); + } + void HexLower(const char* ptr, char c) { + setRequirement(ptr, TFeature::FeatureUpperEncoded); + HexAdd(c - 'a' + 10); + } + void HexAdd(unsigned val) { + HexValue <<= 4; + HexValue += val; + } + void HexReset() { + HexValue = 0; + } + void HexSet(const char* ptr); + + void PctEndImpl(const char* ptr); + void PctEnd(const char* ptr) { + if (nullptr != PctBegin && ptr != PctBegin) + PctEndImpl(ptr); + } + void PctBeg(const char* ptr) { + PctEnd(ptr); + HexReset(); + PctBegin = ptr; + } + + void checkSectionCollision(TField::EField fld1, TField::EField fld2) { + if (Sections[fld1].IsSet() && Sections[fld2].IsSet() && Sections[fld1].Beg == Sections[fld2].Beg) { + Sections[fld1].Reset(); + } + } + + bool doParse(const char* str_beg, size_t length); + TState::EParsed ParseImpl(); + }; } diff --git a/library/cpp/uri/qargs.cpp b/library/cpp/uri/qargs.cpp index 23058f8102..7dd18e7a16 100644 --- a/library/cpp/uri/qargs.cpp +++ b/library/cpp/uri/qargs.cpp @@ -2,32 +2,32 @@ #include <string> namespace NUri { - namespace NOnStackArgsList { - struct TQArgNode { - TQArgNode* Prev; - TQArgNode* Next; - - TStringBuf Name; - TStringBuf Value; - TStringBuf All; - }; - - TQArgNode MakeArg(TQArgNode* prev) { - return {prev, 0, {}, {}, {}}; - } - - const char* SkipDelimiter(const char* str, const char* end) { - while (str != end) - if (*str == '&') - ++str; - else - break; - return str; - } - - /// return next pos or 0 if error - const char* ExtractArgData(const char* pos, const char* end, TQArgNode* arg) { - const char* nameStart = pos; + namespace NOnStackArgsList { + struct TQArgNode { + TQArgNode* Prev; + TQArgNode* Next; + + TStringBuf Name; + TStringBuf Value; + TStringBuf All; + }; + + TQArgNode MakeArg(TQArgNode* prev) { + return {prev, 0, {}, {}, {}}; + } + + const char* SkipDelimiter(const char* str, const char* end) { + while (str != end) + if (*str == '&') + ++str; + else + break; + return str; + } + + /// return next pos or 0 if error + const char* ExtractArgData(const char* pos, const char* end, TQArgNode* arg) { + const char* nameStart = pos; const char* nextArg = strchr(pos, '&'); const char* valueStart = strchr(pos, '='); if (valueStart && nextArg && valueStart < nextArg) // a=1& or a=& @@ -57,223 +57,223 @@ namespace NUri { arg->All = arg->Name; return end; } - } - - // arg can be null - TQArgNode* GetHead(TQArgNode* arg) { - TQArgNode* prev = arg; - while (prev) { - arg = prev; - prev = prev->Prev; - } - return arg; - } - - // arg can be null - TQArgNode* GetLast(TQArgNode* arg) { - TQArgNode* next = arg; - while (next) { - arg = next; - next = arg->Next; - } - return arg; - } - - int CompareName(const TQArgNode* l, const TQArgNode* r) { - return l->Name.compare(r->Name); - } - - TQArgNode* Move(TQArgNode* before, TQArgNode* node) { - TQArgNode* tn = node->Next; - TQArgNode* tp = node->Prev; - - node->Prev = before->Prev; - if (node->Prev) - node->Prev->Next = node; - - node->Next = before; - before->Prev = node; - - if (tn) - tn->Prev = tp; - if (tp) - tp->Next = tn; - - return node; - } - - // return new head - TQArgNode* QSortByName(TQArgNode* iter, TQArgNode* last) { - if (iter == last) - return iter; - if (iter->Next == last) { - int c = CompareName(iter, last); - return c <= 0 ? iter : Move(iter, last); - } else { - TQArgNode* pivot = iter; - iter = iter->Next; - TQArgNode* head = 0; - TQArgNode* tail = 0; - TQArgNode* tailPartitionStart = pivot; - while (true) { - TQArgNode* next = iter->Next; - int c = CompareName(iter, pivot); - int sign = (0 < c) - (c < 0); - switch (sign) { - case -1: - head = head ? Move(head, iter) : Move(pivot, iter); - break; - - case 0: - pivot = Move(pivot, iter); - break; - - case 1: - tail = iter; - break; - } - - if (iter == last) - break; - iter = next; - } - - if (head) - head = QSortByName(head, pivot->Prev); - if (tail) - QSortByName(tailPartitionStart->Next, tail); - return head ? head : pivot; - } + } + + // arg can be null + TQArgNode* GetHead(TQArgNode* arg) { + TQArgNode* prev = arg; + while (prev) { + arg = prev; + prev = prev->Prev; + } + return arg; + } + + // arg can be null + TQArgNode* GetLast(TQArgNode* arg) { + TQArgNode* next = arg; + while (next) { + arg = next; + next = arg->Next; + } + return arg; + } + + int CompareName(const TQArgNode* l, const TQArgNode* r) { + return l->Name.compare(r->Name); + } + + TQArgNode* Move(TQArgNode* before, TQArgNode* node) { + TQArgNode* tn = node->Next; + TQArgNode* tp = node->Prev; + + node->Prev = before->Prev; + if (node->Prev) + node->Prev->Next = node; + + node->Next = before; + before->Prev = node; + + if (tn) + tn->Prev = tp; + if (tp) + tp->Next = tn; + + return node; + } + + // return new head + TQArgNode* QSortByName(TQArgNode* iter, TQArgNode* last) { + if (iter == last) + return iter; + if (iter->Next == last) { + int c = CompareName(iter, last); + return c <= 0 ? iter : Move(iter, last); + } else { + TQArgNode* pivot = iter; + iter = iter->Next; + TQArgNode* head = 0; + TQArgNode* tail = 0; + TQArgNode* tailPartitionStart = pivot; + while (true) { + TQArgNode* next = iter->Next; + int c = CompareName(iter, pivot); + int sign = (0 < c) - (c < 0); + switch (sign) { + case -1: + head = head ? Move(head, iter) : Move(pivot, iter); + break; + + case 0: + pivot = Move(pivot, iter); + break; + + case 1: + tail = iter; + break; + } + + if (iter == last) + break; + iter = next; + } + + if (head) + head = QSortByName(head, pivot->Prev); + if (tail) + QSortByName(tailPartitionStart->Next, tail); + return head ? head : pivot; + } } } - using namespace NOnStackArgsList; - - class TQueryArgProcessing::Pipeline { - public: - Pipeline(TQueryArgProcessing& parent, TUri& subject) - : Parent(parent) - , Subject(subject) - , ArgsCount(0) - , IsDirty(false) - { + using namespace NOnStackArgsList; + + class TQueryArgProcessing::Pipeline { + public: + Pipeline(TQueryArgProcessing& parent, TUri& subject) + : Parent(parent) + , Subject(subject) + , ArgsCount(0) + , IsDirty(false) + { + } + + TQueryArg::EProcessed Process() { + const TStringBuf& query = Subject.GetField(NUri::TField::FieldQuery); + if (query.empty()) + return ProcessEmpty(); + + const char* start = query.data(); + return Parse(start, start + query.length(), 0); + } + + TQueryArg::EProcessed ProcessEmpty() { + if (Parent.Flags & TQueryArg::FeatureRemoveEmptyQuery) + Subject.FldClr(NUri::TField::FieldQuery); + + return TQueryArg::ProcessedOK; + } + + TQueryArg::EProcessed Parse(const char* str, const char* end, TQArgNode* prev) { + str = SkipDelimiter(str, end); + + if (str == end) { + TQArgNode* head = GetHead(prev); + TQArgNode* last = GetLast(prev); + return FinalizeParsing(head, last); + } else { + TQArgNode current = MakeArg(prev); + const char* next = ExtractArgData(str, end, ¤t); + if (!next) + return TQueryArg::ProcessedMalformed; + + TQArgNode* tail = ApplyFilter(prev, ¤t); + + if (++ArgsCount > MaxCount) + return TQueryArg::ProcessedTooMany; + + return Parse(next, end, tail); + } + } + + TQArgNode* ApplyFilter(TQArgNode* prev, TQArgNode* current) { + if (Parent.Flags & TQueryArg::FeatureFilter) { + TQueryArg arg = {current->Name, current->Value}; + if (!Parent.Filter(arg, Parent.FilterData)) { + IsDirty = true; + return prev; + } + } + + if (prev) + prev->Next = current; + return current; } - TQueryArg::EProcessed Process() { - const TStringBuf& query = Subject.GetField(NUri::TField::FieldQuery); - if (query.empty()) - return ProcessEmpty(); - - const char* start = query.data(); - return Parse(start, start + query.length(), 0); - } - - TQueryArg::EProcessed ProcessEmpty() { - if (Parent.Flags & TQueryArg::FeatureRemoveEmptyQuery) - Subject.FldClr(NUri::TField::FieldQuery); - - return TQueryArg::ProcessedOK; - } - - TQueryArg::EProcessed Parse(const char* str, const char* end, TQArgNode* prev) { - str = SkipDelimiter(str, end); - - if (str == end) { - TQArgNode* head = GetHead(prev); - TQArgNode* last = GetLast(prev); - return FinalizeParsing(head, last); - } else { - TQArgNode current = MakeArg(prev); - const char* next = ExtractArgData(str, end, ¤t); - if (!next) - return TQueryArg::ProcessedMalformed; - - TQArgNode* tail = ApplyFilter(prev, ¤t); - - if (++ArgsCount > MaxCount) - return TQueryArg::ProcessedTooMany; - - return Parse(next, end, tail); - } - } - - TQArgNode* ApplyFilter(TQArgNode* prev, TQArgNode* current) { - if (Parent.Flags & TQueryArg::FeatureFilter) { - TQueryArg arg = {current->Name, current->Value}; - if (!Parent.Filter(arg, Parent.FilterData)) { - IsDirty = true; - return prev; - } - } - - if (prev) - prev->Next = current; - return current; - } - - TQueryArg::EProcessed FinalizeParsing(TQArgNode* head, TQArgNode* last) { - if (Parent.Flags & TQueryArg::FeatureSortByName) { - head = QSortByName(head, last); - IsDirty = true; - } - - if (!IsDirty) - return TQueryArg::ProcessedOK; - - bool dirty = Render(head); - - bool rewrite = Parent.Flags & TQueryArg::FeatureRewriteDirty; - if (dirty && rewrite) - Subject.Rewrite(); - return (!dirty || rewrite) ? TQueryArg::ProcessedOK : TQueryArg::ProcessedDirty; - } - - bool Render(TQArgNode* head) { - std::string& result = Parent.Buffer; - result.clear(); - result.reserve(Subject.GetField(NUri::TField::FieldQuery).length()); - bool first = true; - while (head) { - if (first) - first = false; - else - result.append("&"); - - result.append(head->All); - head = head->Next; - } - - if (result.empty()) - return RenderEmpty(); - else - return Subject.FldMemSet(NUri::TField::FieldQuery, result); - } - - bool RenderEmpty() { - if (Parent.Flags & TQueryArg::FeatureRemoveEmptyQuery) - Subject.FldClr(NUri::TField::FieldQuery); - return false; - } - - private: - TQueryArgProcessing& Parent; - TUri& Subject; - - unsigned ArgsCount; - bool IsDirty; - - static const unsigned MaxCount = 100; - }; - - TQueryArgProcessing::TQueryArgProcessing(ui32 flags, TQueryArgFilter filter, void* filterData) - : Flags(flags) - , Filter(filter) - , FilterData(filterData) - { - } - - TQueryArg::EProcessed TQueryArgProcessing::Process(TUri& uri) { - Pipeline pipeline(*this, uri); - return pipeline.Process(); - } + TQueryArg::EProcessed FinalizeParsing(TQArgNode* head, TQArgNode* last) { + if (Parent.Flags & TQueryArg::FeatureSortByName) { + head = QSortByName(head, last); + IsDirty = true; + } + + if (!IsDirty) + return TQueryArg::ProcessedOK; + + bool dirty = Render(head); + + bool rewrite = Parent.Flags & TQueryArg::FeatureRewriteDirty; + if (dirty && rewrite) + Subject.Rewrite(); + return (!dirty || rewrite) ? TQueryArg::ProcessedOK : TQueryArg::ProcessedDirty; + } + + bool Render(TQArgNode* head) { + std::string& result = Parent.Buffer; + result.clear(); + result.reserve(Subject.GetField(NUri::TField::FieldQuery).length()); + bool first = true; + while (head) { + if (first) + first = false; + else + result.append("&"); + + result.append(head->All); + head = head->Next; + } + + if (result.empty()) + return RenderEmpty(); + else + return Subject.FldMemSet(NUri::TField::FieldQuery, result); + } + + bool RenderEmpty() { + if (Parent.Flags & TQueryArg::FeatureRemoveEmptyQuery) + Subject.FldClr(NUri::TField::FieldQuery); + return false; + } + + private: + TQueryArgProcessing& Parent; + TUri& Subject; + + unsigned ArgsCount; + bool IsDirty; + + static const unsigned MaxCount = 100; + }; + + TQueryArgProcessing::TQueryArgProcessing(ui32 flags, TQueryArgFilter filter, void* filterData) + : Flags(flags) + , Filter(filter) + , FilterData(filterData) + { + } + + TQueryArg::EProcessed TQueryArgProcessing::Process(TUri& uri) { + Pipeline pipeline(*this, uri); + return pipeline.Process(); + } } diff --git a/library/cpp/uri/qargs.h b/library/cpp/uri/qargs.h index fcba7cbd0c..5a22330549 100644 --- a/library/cpp/uri/qargs.h +++ b/library/cpp/uri/qargs.h @@ -5,18 +5,18 @@ #include <string> namespace NUri { - class TQueryArgProcessing { - public: - TQueryArgProcessing(ui32 flags, TQueryArgFilter filter = 0, void* filterData = 0); + class TQueryArgProcessing { + public: + TQueryArgProcessing(ui32 flags, TQueryArgFilter filter = 0, void* filterData = 0); - TQueryArg::EProcessed Process(TUri& uri); + TQueryArg::EProcessed Process(TUri& uri); - private: - ui32 Flags; - TQueryArgFilter Filter; - void* FilterData; + private: + ui32 Flags; + TQueryArgFilter Filter; + void* FilterData; - class Pipeline; - std::string Buffer; - }; + class Pipeline; + std::string Buffer; + }; } diff --git a/library/cpp/uri/uri-ru_ut.cpp b/library/cpp/uri/uri-ru_ut.cpp index ec35a164d2..a7306cbc64 100644 --- a/library/cpp/uri/uri-ru_ut.cpp +++ b/library/cpp/uri/uri-ru_ut.cpp @@ -1,7 +1,7 @@ #include "uri_ut.h" #include <library/cpp/charset/recyr.hh> #include <library/cpp/html/entity/htmlentity.h> -#include <util/system/maxlen.h> +#include <util/system/maxlen.h> namespace NUri { namespace { @@ -15,137 +15,137 @@ namespace NUri { Y_UNIT_TEST_SUITE(URLTestRU) { Y_UNIT_TEST(test_httpURL2) { - TUri url; - UNIT_ASSERT_VALUES_EQUAL(url.Parse("g:h"), TState::ParsedBadScheme); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("http:g"), TState::ParsedBadFormat); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("/../g"), TState::ParsedBadPath); - const char* const UpCaseUrl = "http://www.TEST.Ru:80/InDex.html"; - UNIT_ASSERT_VALUES_EQUAL(url.Parse(UpCaseUrl), TState::ParsedOK); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://www.TEST.Ru/InDex.html"); - UNIT_ASSERT_VALUES_EQUAL(url.Parse(UpCaseUrl, TFeature::FeaturesDefault | TFeature::FeatureToLower), TState::ParsedOK); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://www.test.ru/InDex.html"); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagScheme), "http:"); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagScheme | TField::FlagHost), "http://www.test.ru"); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagHost), "www.test.ru"); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagHost | TField::FlagPath), "www.test.ru/InDex.html"); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagQuery), ""); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("http://www.TEST.Ru:90/InDex.html"), TState::ParsedOK); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagHostPort | TField::FlagPath), "www.TEST.Ru:90/InDex.html"); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("www.ya.ru/index.html"), TState::ParsedOK); - UNIT_ASSERT(!url.IsValidAbs()); - UNIT_ASSERT(url.IsNull(TField::FlagHost)); - UNIT_ASSERT(!url.IsNull(TField::FlagPath)); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagPath), "www.ya.ru/index.html"); + TUri url; + UNIT_ASSERT_VALUES_EQUAL(url.Parse("g:h"), TState::ParsedBadScheme); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("http:g"), TState::ParsedBadFormat); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("/../g"), TState::ParsedBadPath); + const char* const UpCaseUrl = "http://www.TEST.Ru:80/InDex.html"; + UNIT_ASSERT_VALUES_EQUAL(url.Parse(UpCaseUrl), TState::ParsedOK); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://www.TEST.Ru/InDex.html"); + UNIT_ASSERT_VALUES_EQUAL(url.Parse(UpCaseUrl, TFeature::FeaturesDefault | TFeature::FeatureToLower), TState::ParsedOK); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://www.test.ru/InDex.html"); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagScheme), "http:"); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagScheme | TField::FlagHost), "http://www.test.ru"); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagHost), "www.test.ru"); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagHost | TField::FlagPath), "www.test.ru/InDex.html"); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagQuery), ""); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("http://www.TEST.Ru:90/InDex.html"), TState::ParsedOK); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagHostPort | TField::FlagPath), "www.TEST.Ru:90/InDex.html"); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("www.ya.ru/index.html"), TState::ParsedOK); + UNIT_ASSERT(!url.IsValidAbs()); + UNIT_ASSERT(url.IsNull(TField::FlagHost)); + UNIT_ASSERT(!url.IsNull(TField::FlagPath)); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagPath), "www.ya.ru/index.html"); UNIT_ASSERT_VALUES_EQUAL(url.Parse(AsWin1251("www.TEST.Ru/ФЕУФ\\'\".html?ФЕУФ\\'\"=ФЕУФ+\\'\"%10")), TState::ParsedOK); UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), AsWin1251("www.TEST.Ru/ФЕУФ\\'\".html?ФЕУФ\\'\"=ФЕУФ+\\'\"%10")); UNIT_ASSERT_VALUES_EQUAL(url.Parse(AsWin1251("www.TEST.Ru/ФЕУФ\\'\".html?ФЕУФ\\'\"=ФЕУФ+\\'\"%10"), - TFeature::FeaturesDefault | TFeature::FeatureEncodeExtendedASCII), - TState::ParsedOK); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), + TFeature::FeaturesDefault | TFeature::FeatureEncodeExtendedASCII), + TState::ParsedOK); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), AsWin1251("www.TEST.Ru/%D4%C5%D3%D4\\'\".html?%D4%C5%D3%D4\\'\"=%D4%C5%D3%D4+\\'\"%10")); UNIT_ASSERT_VALUES_EQUAL(url.Parse(AsWin1251("www.TEST.Ru/ФЕУФ\\'\".html?ФЕУФ\\'\"=ФЕУФ+\\'\"%10"), - TFeature::FeaturesDefault | TFeature::FeatureEncodeForSQL), - TState::ParsedOK); + TFeature::FeaturesDefault | TFeature::FeatureEncodeForSQL), + TState::ParsedOK); UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), AsWin1251("www.TEST.Ru/ФЕУФ%5C%27%22.html?ФЕУФ%5C%27%22=ФЕУФ+%5C%27%22%10")); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("q/%33%26%13%2f%2b%30%20", - TFeature::FeaturesDefault | TFeature::FeatureDecodeStandard), - TState::ParsedOK); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "q/3%26%13/%2B0%20"); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("q/%33%26%13%2f%2b%30%20", + TFeature::FeaturesDefault | TFeature::FeatureDecodeStandard), + TState::ParsedOK); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "q/3%26%13/%2B0%20"); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("http://www.prime-tass.ru/news/0/{656F5BAE-6677-4762-9BED-9E3B77E72055}.uif"), - TState::ParsedOK); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("//server/path"), TState::ParsedOK); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("//server/path", TFeature::FeaturesRobot), TState::ParsedOK); - } + UNIT_ASSERT_VALUES_EQUAL(url.Parse("http://www.prime-tass.ru/news/0/{656F5BAE-6677-4762-9BED-9E3B77E72055}.uif"), + TState::ParsedOK); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("//server/path"), TState::ParsedOK); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("//server/path", TFeature::FeaturesRobot), TState::ParsedOK); + } const TString links[] = { - "viewforum.php?f=1&sid=b4568481b67b1d7683bea78634b2e240", "viewforum.php?f=1&sid=b4568481b67b1d7683bea78634b2e240", - "./viewtopic.php?p=74&sid=6#p74", "./viewtopic.php?p=74&sid=6#p74", - "viewtopic.php?p=9313&sid=8#9313", "viewtopic.php?p=9313&sid=8#9313", - "profile.php?mode=viewprofile&u=-1#drafts&sid=a6e5989cee27adb5996bfff044af04ca", "profile.php?mode=viewprofile&u=-1#drafts&sid=a6e5989cee27adb5996bfff044af04ca", + "viewforum.php?f=1&sid=b4568481b67b1d7683bea78634b2e240", "viewforum.php?f=1&sid=b4568481b67b1d7683bea78634b2e240", + "./viewtopic.php?p=74&sid=6#p74", "./viewtopic.php?p=74&sid=6#p74", + "viewtopic.php?p=9313&sid=8#9313", "viewtopic.php?p=9313&sid=8#9313", + "profile.php?mode=viewprofile&u=-1#drafts&sid=a6e5989cee27adb5996bfff044af04ca", "profile.php?mode=viewprofile&u=-1#drafts&sid=a6e5989cee27adb5996bfff044af04ca", - "images\nil.jpg", "images%0Ail.jpg", - "http://caedebaturque.termez.su\r\n/?article=218", "http://caedebaturque.termez.su%0D%0A/?article=218", + "images\nil.jpg", "images%0Ail.jpg", + "http://caedebaturque.termez.su\r\n/?article=218", "http://caedebaturque.termez.su%0D%0A/?article=218", AsKoi8("javascript:window.external.AddFavorite(\'http://www.humor.look.ru/\',\'Злобные Деды Морозы!!!\')"), "javascript:window.external.AddFavorite(\'http://www.humor.look.ru/\',\'%FA%CC%CF%C2%CE%D9%C5%20%E4%C5%C4%D9%20%ED%CF%D2%CF%DA%D9!!!\')", - "search.php?search_author=%CB%FE%E4%EC%E8%EB%E0+%C3%F3%F1%E5%E2%E0&showresults=posts&sid=8", "search.php?search_author=%CB%FE%E4%EC%E8%EB%E0+%C3%F3%F1%E5%E2%E0&showresults=posts&sid=8", + "search.php?search_author=%CB%FE%E4%EC%E8%EB%E0+%C3%F3%F1%E5%E2%E0&showresults=posts&sid=8", "search.php?search_author=%CB%FE%E4%EC%E8%EB%E0+%C3%F3%F1%E5%E2%E0&showresults=posts&sid=8", AsWin1251("/Search/author/?q=Штрибель Х.В."), "/Search/author/?q=%D8%F2%F0%E8%E1%E5%EB%FC%20%D5.%C2.", AsWin1251("javascript:ins(\'ГОРШОК\')"), "javascript:ins(\'%C3%CE%D0%D8%CE%CA\')", AsWin1251("?l=я"), "?l=%FF", AsWin1251("content.php?id=3392&theme=Цена"), "content.php?id=3392&theme=%D6%E5%ED%E0", - "/a-mp3/stype-1/?search=А", "/a-mp3/stype-1/?search=%D0%90", - "/a-mp3/stype-1/?search=Б", "/a-mp3/stype-1/?search=%D0%91", - "/a-mp3/stype-1/?search=В", "/a-mp3/stype-1/?search=%D0%92", - "/a-mp3/stype-1/?search=Г", "/a-mp3/stype-1/?search=%D0%93", - "/a-mp3/stype-1/?search=Д", "/a-mp3/stype-1/?search=%D0%94", - "/a-mp3/stype-1/?search=Е", "/a-mp3/stype-1/?search=%D0%95", - "/a-mp3/stype-1/?search=Ж", "/a-mp3/stype-1/?search=%D0%96", - "/a-mp3/stype-1/?search=З", "/a-mp3/stype-1/?search=%D0%97", - // %98 is not defined in CP1251 so don't put it here explicitly + "/a-mp3/stype-1/?search=А", "/a-mp3/stype-1/?search=%D0%90", + "/a-mp3/stype-1/?search=Б", "/a-mp3/stype-1/?search=%D0%91", + "/a-mp3/stype-1/?search=В", "/a-mp3/stype-1/?search=%D0%92", + "/a-mp3/stype-1/?search=Г", "/a-mp3/stype-1/?search=%D0%93", + "/a-mp3/stype-1/?search=Д", "/a-mp3/stype-1/?search=%D0%94", + "/a-mp3/stype-1/?search=Е", "/a-mp3/stype-1/?search=%D0%95", + "/a-mp3/stype-1/?search=Ж", "/a-mp3/stype-1/?search=%D0%96", + "/a-mp3/stype-1/?search=З", "/a-mp3/stype-1/?search=%D0%97", + // %98 is not defined in CP1251 so don't put it here explicitly "/a-mp3/stype-1/?search=\xD0\x98", "/a-mp3/stype-1/?search=%D0%98", - "/a-mp3/stype-1/?search=Й", "/a-mp3/stype-1/?search=%D0%99", - "/a-mp3/stype-1/?search=К", "/a-mp3/stype-1/?search=%D0%9A", - "/a-mp3/stype-1/?search=Л", "/a-mp3/stype-1/?search=%D0%9B", - "/a-mp3/stype-1/?search=М", "/a-mp3/stype-1/?search=%D0%9C", - "/a-mp3/stype-1/?search=Н", "/a-mp3/stype-1/?search=%D0%9D", - "/a-mp3/stype-1/?search=О", "/a-mp3/stype-1/?search=%D0%9E", - "/a-mp3/stype-1/?search=П", "/a-mp3/stype-1/?search=%D0%9F", + "/a-mp3/stype-1/?search=Й", "/a-mp3/stype-1/?search=%D0%99", + "/a-mp3/stype-1/?search=К", "/a-mp3/stype-1/?search=%D0%9A", + "/a-mp3/stype-1/?search=Л", "/a-mp3/stype-1/?search=%D0%9B", + "/a-mp3/stype-1/?search=М", "/a-mp3/stype-1/?search=%D0%9C", + "/a-mp3/stype-1/?search=Н", "/a-mp3/stype-1/?search=%D0%9D", + "/a-mp3/stype-1/?search=О", "/a-mp3/stype-1/?search=%D0%9E", + "/a-mp3/stype-1/?search=П", "/a-mp3/stype-1/?search=%D0%9F", "/a-mp3/stype-1/?search=\xD0", "/a-mp3/stype-1/?search=%D0", - "/a-mp3/stype-1/?search=С", "/a-mp3/stype-1/?search=%D0%A1", - "/a-mp3/stype-1/?search=Т", "/a-mp3/stype-1/?search=%D0%A2", - "/a-mp3/stype-1/?search=У", "/a-mp3/stype-1/?search=%D0%A3", - "/a-mp3/stype-1/?search=Ф", "/a-mp3/stype-1/?search=%D0%A4", - "/a-mp3/stype-1/?search=Х", "/a-mp3/stype-1/?search=%D0%A5", - "/a-mp3/stype-1/?search=Ц", "/a-mp3/stype-1/?search=%D0%A6", - "/a-mp3/stype-1/?search=Ч", "/a-mp3/stype-1/?search=%D0%A7", - "/a-mp3/stype-1/?search=Ш", "/a-mp3/stype-1/?search=%D0%A8", - "/a-mp3/stype-1/?search=Щ", "/a-mp3/stype-1/?search=%D0%A9", - "/a-mp3/stype-1/?search=Ы", "/a-mp3/stype-1/?search=%D0%AB", - "/a-mp3/stype-1/?search=Э", "/a-mp3/stype-1/?search=%D0%AD", - "/a-mp3/stype-1/?search=Ю", "/a-mp3/stype-1/?search=%D0%AE", - "/a-mp3/stype-1/?search=Я", "/a-mp3/stype-1/?search=%D0%AF", - - "javascript:emoticon(\":'(\")", "javascript:emoticon(\":\'(\")", - "javascript:emoticon(\'>:o\')", "javascript:emoticon(\'>:o\')", - "javascript:emoticon(\']:->\')", "javascript:emoticon(\']:->\')", - "javascript:emoticon(\':-!\')", "javascript:emoticon(\':-!\')", - "javascript:emoticon(\'@}->--\')", "javascript:emoticon(\'@}->--\')", - "http://www.is-ufa.ru/price2/price_IS.rar", "http://www.is-ufa.ru/price2/price_IS.rar", - "mailto:info@etem.de", "mailto:info@etem.de", - ""http://www.fubix.ru"", "\"http://www.fubix.ru\"", + "/a-mp3/stype-1/?search=С", "/a-mp3/stype-1/?search=%D0%A1", + "/a-mp3/stype-1/?search=Т", "/a-mp3/stype-1/?search=%D0%A2", + "/a-mp3/stype-1/?search=У", "/a-mp3/stype-1/?search=%D0%A3", + "/a-mp3/stype-1/?search=Ф", "/a-mp3/stype-1/?search=%D0%A4", + "/a-mp3/stype-1/?search=Х", "/a-mp3/stype-1/?search=%D0%A5", + "/a-mp3/stype-1/?search=Ц", "/a-mp3/stype-1/?search=%D0%A6", + "/a-mp3/stype-1/?search=Ч", "/a-mp3/stype-1/?search=%D0%A7", + "/a-mp3/stype-1/?search=Ш", "/a-mp3/stype-1/?search=%D0%A8", + "/a-mp3/stype-1/?search=Щ", "/a-mp3/stype-1/?search=%D0%A9", + "/a-mp3/stype-1/?search=Ы", "/a-mp3/stype-1/?search=%D0%AB", + "/a-mp3/stype-1/?search=Э", "/a-mp3/stype-1/?search=%D0%AD", + "/a-mp3/stype-1/?search=Ю", "/a-mp3/stype-1/?search=%D0%AE", + "/a-mp3/stype-1/?search=Я", "/a-mp3/stype-1/?search=%D0%AF", + + "javascript:emoticon(\":'(\")", "javascript:emoticon(\":\'(\")", + "javascript:emoticon(\'>:o\')", "javascript:emoticon(\'>:o\')", + "javascript:emoticon(\']:->\')", "javascript:emoticon(\']:->\')", + "javascript:emoticon(\':-!\')", "javascript:emoticon(\':-!\')", + "javascript:emoticon(\'@}->--\')", "javascript:emoticon(\'@}->--\')", + "http://www.is-ufa.ru/price2/price_IS.rar", "http://www.is-ufa.ru/price2/price_IS.rar", + "mailto:info@etem.de", "mailto:info@etem.de", + ""http://www.fubix.ru"", "\"http://www.fubix.ru\"", AsWin1251("mailto:kampa@ukr.net?subject=Арабский язык"), "mailto:kampa@ukr.net?subject=%C0%F0%E0%E1%F1%EA%E8%E9%20%FF%E7%FB%EA", {}}; Y_UNIT_TEST(testHtLinkDecode) { - char decodedlink[URL_MAXLEN + 10]; - for (int i = 0; links[i]; i += 2) { + char decodedlink[URL_MAXLEN + 10]; + for (int i = 0; links[i]; i += 2) { UNIT_ASSERT(HtLinkDecode(links[i].c_str(), decodedlink, sizeof(decodedlink))); UNIT_ASSERT_VALUES_EQUAL(decodedlink, links[i + 1]); - } - } + } + } Y_UNIT_TEST(testRuIDNA) { - { + { #define DEC "\xD7\xE5\xF0\xE5\xEf\xEE\xE2\xE5\xF6.\xF0\xF4" /* "Череповец.рф" in Windows-1251 */ #define ENC "%D7%E5%F0%E5%EF%EE%E2%E5%F6.%F0%F4" // punycode corresponds to lowercase #define PNC "xn--b1afab7bff7cb.xn--p1ai" - TTest test = { - "http://" ENC "/" ENC "?" ENC "#" ENC, TParseFlags(TFeature::FeaturesAll | TFeature::FeatureAllowHostIDN, TFeature::FeatureDecodeExtendedASCII), TState::ParsedOK, "http", "", "", DEC, 80, "/" ENC, ENC, ENC}; - TUri url; - URL_TEST_ENC(url, test, CODES_WIN); - UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::FieldHostAscii), PNC); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" DEC "/" ENC "?" ENC "#" ENC); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagHostAscii), "http://" PNC "/" ENC "?" ENC "#" ENC); + TTest test = { + "http://" ENC "/" ENC "?" ENC "#" ENC, TParseFlags(TFeature::FeaturesAll | TFeature::FeatureAllowHostIDN, TFeature::FeatureDecodeExtendedASCII), TState::ParsedOK, "http", "", "", DEC, 80, "/" ENC, ENC, ENC}; + TUri url; + URL_TEST_ENC(url, test, CODES_WIN); + UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::FieldHostAscii), PNC); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" DEC "/" ENC "?" ENC "#" ENC); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagHostAscii), "http://" PNC "/" ENC "?" ENC "#" ENC); #undef PNC #undef DEC #undef ENC - } - } + } + } // Regression test for SEARCH-11283 Y_UNIT_TEST(RegressionTest11283) { diff --git a/library/cpp/uri/uri.cpp b/library/cpp/uri/uri.cpp index 56a9a4e5ef..55cfe57e28 100644 --- a/library/cpp/uri/uri.cpp +++ b/library/cpp/uri/uri.cpp @@ -1,621 +1,621 @@ #include "uri.h" #include "parse.h" - + #include <util/string/cast.h> #include <util/string/util.h> #include <util/system/yassert.h> - + namespace NUri { - TState::EParsed TUri::CheckHost(const TStringBuf& host) { + TState::EParsed TUri::CheckHost(const TStringBuf& host) { if (host.empty()) - return ParsedOK; - - unsigned domainLevel = 0; - unsigned domainLevelOfUnderscore = 0; - - bool isAlnum = false; - bool startLabel = true; - for (size_t i = 0; i != host.length(); ++i) { - const char ch = host[i]; - - if ('.' == ch) { // label separator - if (!isAlnum || startLabel) // previous label must end in alnum - return ParsedBadHost; - startLabel = true; - continue; - } - - isAlnum = isalnum((const unsigned char)ch); - - if (startLabel) { // label is starting - if (!isAlnum && '_' != ch) // new label must start with alnum or '_' - return ParsedBadHost; - startLabel = false; - ++domainLevel; - if (ch == '_') - domainLevelOfUnderscore = domainLevel; - continue; - } - - if (isAlnum || '-' == ch) - continue; - - if (ch == '_') { // non-standard case we allow for certain hosts + return ParsedOK; + + unsigned domainLevel = 0; + unsigned domainLevelOfUnderscore = 0; + + bool isAlnum = false; + bool startLabel = true; + for (size_t i = 0; i != host.length(); ++i) { + const char ch = host[i]; + + if ('.' == ch) { // label separator + if (!isAlnum || startLabel) // previous label must end in alnum + return ParsedBadHost; + startLabel = true; + continue; + } + + isAlnum = isalnum((const unsigned char)ch); + + if (startLabel) { // label is starting + if (!isAlnum && '_' != ch) // new label must start with alnum or '_' + return ParsedBadHost; + startLabel = false; + ++domainLevel; + if (ch == '_') + domainLevelOfUnderscore = domainLevel; + continue; + } + + if (isAlnum || '-' == ch) + continue; + + if (ch == '_') { // non-standard case we allow for certain hosts domainLevelOfUnderscore = domainLevel; - continue; - } - - return ParsedBadHost; + continue; + } + + return ParsedBadHost; } - if (0 < domainLevelOfUnderscore && domainLevel < 2 + domainLevelOfUnderscore) - return ParsedBadHost; - - return ParsedOK; - } - - /********************************************************/ - TUri::TUri(const TStringBuf& host, ui16 port, const TStringBuf& path, const TStringBuf& query, const TStringBuf& scheme, unsigned defaultPort) - : FieldsSet(0) - , Port(port) - , DefaultPort(0) - , Scheme(SchemeEmpty) - , FieldsDirty(0) - { + if (0 < domainLevelOfUnderscore && domainLevel < 2 + domainLevelOfUnderscore) + return ParsedBadHost; + + return ParsedOK; + } + + /********************************************************/ + TUri::TUri(const TStringBuf& host, ui16 port, const TStringBuf& path, const TStringBuf& query, const TStringBuf& scheme, unsigned defaultPort) + : FieldsSet(0) + , Port(port) + , DefaultPort(0) + , Scheme(SchemeEmpty) + , FieldsDirty(0) + { if (!scheme.empty()) { if (SetSchemeImpl(TSchemeInfo::Get(scheme)).Str.empty()) - FldSet(FieldScheme, scheme); + FldSet(FieldScheme, scheme); } - if (0 < defaultPort) // override the scheme's default port - DefaultPort = static_cast<ui16>(defaultPort); + if (0 < defaultPort) // override the scheme's default port + DefaultPort = static_cast<ui16>(defaultPort); - char sport[6]; // enough for ui16 - if (0 != port) { - const size_t len = ToString(port, sport, sizeof(sport)); - FldSet(FieldPort, TStringBuf(sport, len)); - } + char sport[6]; // enough for ui16 + if (0 != port) { + const size_t len = ToString(port, sport, sizeof(sport)); + FldSet(FieldPort, TStringBuf(sport, len)); + } - FldTrySet(FieldHost, host); - FldTrySet(FieldPath, path); - FldTrySet(FieldQuery, query); + FldTrySet(FieldHost, host); + FldTrySet(FieldPath, path); + FldTrySet(FieldQuery, query); - Rewrite(); + Rewrite(); } - /********************************************************/ - bool TUri::FldSetImpl( - EField field, TStringBuf value, bool strconst, bool nocopy) { - if (!FldIsValid(field)) - return false; + /********************************************************/ + bool TUri::FldSetImpl( + EField field, TStringBuf value, bool strconst, bool nocopy) { + if (!FldIsValid(field)) + return false; - switch (field) { - case FieldScheme: + switch (field) { + case FieldScheme: if (!SetScheme(TSchemeInfo::Get(value)).Str.empty()) - return false; - break; + return false; + break; - case FieldPort: + case FieldPort: Port = value.empty() ? 0 : FromString<ui16>(value); - break; + break; - default: - break; - } + default: + break; + } - if (!value.IsInited()) { - FldClr(field); - return false; - } + if (!value.IsInited()) { + FldClr(field); + return false; + } - if (strconst) { // string constants don't need to be saved in the buffer - FldMarkClean(field); - FldSetNoDirty(field, value); + if (strconst) { // string constants don't need to be saved in the buffer + FldMarkClean(field); + FldSetNoDirty(field, value); return false; - } + } - if (nocopy) { - FldSet(field, value); - return true; - } + if (nocopy) { + FldSet(field, value); + return true; + } - return FldTryCpy(field, value); + return FldTryCpy(field, value); } - /********************************************************/ - bool TUri::FldTryCpy(EField field, const TStringBuf& value) { - if (!FldIsDirty(field)) { - do { + /********************************************************/ + bool TUri::FldTryCpy(EField field, const TStringBuf& value) { + if (!FldIsDirty(field)) { + do { if (!FldIsSet(field)) - break; - - TStringBuf& fld = Fields[field]; - if (fld.length() < value.length()) - break; - - char* oldV = (char*)fld.data(); - if (!IsInBuffer(oldV)) - break; - - memcpy(oldV, value.data(), value.length()); - oldV[value.length()] = 0; - fld.Trunc(value.length()); - return false; - } while (false); - - FldMarkDirty(field); - } - + break; + + TStringBuf& fld = Fields[field]; + if (fld.length() < value.length()) + break; + + char* oldV = (char*)fld.data(); + if (!IsInBuffer(oldV)) + break; + + memcpy(oldV, value.data(), value.length()); + oldV[value.length()] = 0; + fld.Trunc(value.length()); + return false; + } while (false); + + FldMarkDirty(field); + } + FldSetNoDirty(field, value); return true; } - /********************************************************/ - void TUri::RewriteImpl() { - size_t len = 0; - for (int i = 0; i < FieldAllMAX; ++i) { - const EField fld = EField(i); - if (FldIsSet(fld)) - len += 1 + Fields[fld].length(); - } + /********************************************************/ + void TUri::RewriteImpl() { + size_t len = 0; + for (int i = 0; i < FieldAllMAX; ++i) { + const EField fld = EField(i); + if (FldIsSet(fld)) + len += 1 + Fields[fld].length(); + } - if (!len) + if (!len) Buffer.Clear(); - else { + else { TBuffer newbuf; newbuf.Resize(len); TMemoryWriteBuffer out(newbuf.data(), newbuf.size()); - for (int i = 0; i < FieldAllMAX; ++i) { - const EField fld = EField(i); - if (!FldIsSet(fld)) - continue; - - const char* beg = out.Buf(); - const TStringBuf& val = Fields[fld]; - out << val; - FldSetNoDirty(fld, TStringBuf(beg, val.length())); + for (int i = 0; i < FieldAllMAX; ++i) { + const EField fld = EField(i); + if (!FldIsSet(fld)) + continue; + + const char* beg = out.Buf(); + const TStringBuf& val = Fields[fld]; + out << val; + FldSetNoDirty(fld, TStringBuf(beg, val.length())); out << '\0'; - } + } Buffer = std::move(newbuf); } - CheckMissingFields(); + CheckMissingFields(); - FieldsDirty = 0; + FieldsDirty = 0; } - void TUri::CheckMissingFields() { - // if host is set but path is not... - if (FldSetCmp(FlagPath | FlagHost, FlagHost)) - // ... and the scheme requires a path... - if (GetSchemeInfo().FldReq & FlagPath) - // ... set path + void TUri::CheckMissingFields() { + // if host is set but path is not... + if (FldSetCmp(FlagPath | FlagHost, FlagHost)) + // ... and the scheme requires a path... + if (GetSchemeInfo().FldReq & FlagPath) + // ... set path FldSetNoDirty(FieldPath, TStringBuf("/")); } - /********************************************************/ - void TUri::Merge(const TUri& base, int correctAbs) { - if (base.Scheme == SchemeUnknown) - return; + /********************************************************/ + void TUri::Merge(const TUri& base, int correctAbs) { + if (base.Scheme == SchemeUnknown) + return; - if (!base.IsValidGlobal()) - return; + if (!base.IsValidGlobal()) + return; - const TStringBuf& selfscheme = GetField(FieldScheme); - // basescheme is present since IsValidGlobal() succeeded - const TStringBuf& basescheme = base.GetField(FieldScheme); - const bool noscheme = !selfscheme.IsInited(); - if (!noscheme && !EqualNoCase(selfscheme, basescheme)) - return; + const TStringBuf& selfscheme = GetField(FieldScheme); + // basescheme is present since IsValidGlobal() succeeded + const TStringBuf& basescheme = base.GetField(FieldScheme); + const bool noscheme = !selfscheme.IsInited(); + if (!noscheme && !EqualNoCase(selfscheme, basescheme)) + return; - const ui32 cleanFields = ~FieldsDirty; - do { + const ui32 cleanFields = ~FieldsDirty; + do { static constexpr TStringBuf rootPath = "/"; - if (noscheme) { + if (noscheme) { if (!basescheme.empty()) { - FldSetNoDirty(FieldScheme, basescheme); - // check if it is canonical - if (basescheme.data() != base.GetSchemeInfo().Str.data()) - FldMarkDirty(FieldScheme); - } - Scheme = base.Scheme; - DefaultPort = base.DefaultPort; - } - - if (!IsNull(FlagHost)) - break; // no merge - - FldTrySet(FieldHost, base); - FldChkSet(FieldPort, base); - Port = base.Port; - - if (noscheme && IsNull(FlagQuery) && IsNull(FlagPath)) - FldTrySet(FieldQuery, base); - - if (IsNull(FlagAuth) && !base.IsNull(FlagAuth)) { - FldChkSet(FieldUser, base); - FldChkSet(FieldPass, base); + FldSetNoDirty(FieldScheme, basescheme); + // check if it is canonical + if (basescheme.data() != base.GetSchemeInfo().Str.data()) + FldMarkDirty(FieldScheme); + } + Scheme = base.Scheme; + DefaultPort = base.DefaultPort; + } + + if (!IsNull(FlagHost)) + break; // no merge + + FldTrySet(FieldHost, base); + FldChkSet(FieldPort, base); + Port = base.Port; + + if (noscheme && IsNull(FlagQuery) && IsNull(FlagPath)) + FldTrySet(FieldQuery, base); + + if (IsNull(FlagAuth) && !base.IsNull(FlagAuth)) { + FldChkSet(FieldUser, base); + FldChkSet(FieldPass, base); } - if (IsValidAbs()) - break; + if (IsValidAbs()) + break; - TStringBuf p0 = base.GetField(FieldPath); - if (!p0.IsInited()) - p0 = rootPath; + TStringBuf p0 = base.GetField(FieldPath); + if (!p0.IsInited()) + p0 = rootPath; - TStringBuf p1 = GetField(FieldPath); - if (!p1.IsInited()) { - if (p0.data() != rootPath.data()) - FldSet(FieldPath, p0); - else - FldSetNoDirty(FieldPath, rootPath); - break; - } - if (p1 && '/' == p1[0]) - p1.Skip(1); // p0 will have one + TStringBuf p1 = GetField(FieldPath); + if (!p1.IsInited()) { + if (p0.data() != rootPath.data()) + FldSet(FieldPath, p0); + else + FldSetNoDirty(FieldPath, rootPath); + break; + } + if (p1 && '/' == p1[0]) + p1.Skip(1); // p0 will have one - bool pathop = true; + bool pathop = true; TTempBufOutput out(p0.length() + p1.length() + 4); - out << p0; - if ('/' != p0.back()) - out << "/../"; - else if (p1.empty() || '.' != p1[0]) - pathop = false; - out << p1; - - char* beg = out.Data(); - char* end = beg + out.Filled(); - if (pathop && !PathOperation(beg, end, correctAbs)) { - Clear(); - break; - } - - // Needs immediate forced rewrite because of TTempBuf - FldSetNoDirty(FieldPath, TStringBuf(beg, end)); - RewriteImpl(); - } while (false); - - CheckMissingFields(); - - // rewrite only if borrowed fields from base - if (cleanFields & FieldsDirty) - RewriteImpl(); + out << p0; + if ('/' != p0.back()) + out << "/../"; + else if (p1.empty() || '.' != p1[0]) + pathop = false; + out << p1; + + char* beg = out.Data(); + char* end = beg + out.Filled(); + if (pathop && !PathOperation(beg, end, correctAbs)) { + Clear(); + break; + } + + // Needs immediate forced rewrite because of TTempBuf + FldSetNoDirty(FieldPath, TStringBuf(beg, end)); + RewriteImpl(); + } while (false); + + CheckMissingFields(); + + // rewrite only if borrowed fields from base + if (cleanFields & FieldsDirty) + RewriteImpl(); } - /********************************************************/ - TUri::TLinkType TUri::Normalize(const TUri& base, - const TStringBuf& link, const TStringBuf& codebase, long careFlags, ECharset enc) { - // parse URL - if (ParsedOK != ParseImpl(link, careFlags, 0, SchemeEmpty, enc)) - return LinkIsBad; + /********************************************************/ + TUri::TLinkType TUri::Normalize(const TUri& base, + const TStringBuf& link, const TStringBuf& codebase, long careFlags, ECharset enc) { + // parse URL + if (ParsedOK != ParseImpl(link, careFlags, 0, SchemeEmpty, enc)) + return LinkIsBad; - const TStringBuf& host = GetHost(); + const TStringBuf& host = GetHost(); - // merge with base URL - // taken either from _BASE_ property or from optional argument + // merge with base URL + // taken either from _BASE_ property or from optional argument if (!codebase.empty()) { - // if optional code base given -- parse it - TUri codebaseUrl; - if (codebaseUrl.ParseImpl(codebase, careFlags, 0, SchemeEmpty, enc) != ParsedOK || !codebaseUrl.IsValidAbs()) - return LinkIsBad; - Merge(codebaseUrl); - } else { - // Base is already in this variable - // see SetProperty() for details - Merge(base); - } - - // check result: must be correct absolute URL - if (!IsValidAbs()) - return LinkBadAbs; + // if optional code base given -- parse it + TUri codebaseUrl; + if (codebaseUrl.ParseImpl(codebase, careFlags, 0, SchemeEmpty, enc) != ParsedOK || !codebaseUrl.IsValidAbs()) + return LinkIsBad; + Merge(codebaseUrl); + } else { + // Base is already in this variable + // see SetProperty() for details + Merge(base); + } + + // check result: must be correct absolute URL + if (!IsValidAbs()) + return LinkBadAbs; if (!host.empty()) { - // - we don't care about different ports for the same server - // - we don't care about win|www|koi|etc. preffixes for the same server - if (GetPort() != base.GetPort() || !EqualNoCase(host, base.GetHost())) - return LinkIsGlobal; - } + // - we don't care about different ports for the same server + // - we don't care about win|www|koi|etc. preffixes for the same server + if (GetPort() != base.GetPort() || !EqualNoCase(host, base.GetHost())) + return LinkIsGlobal; + } - // find out if it is link to itself then ignore it - if (!Compare(base, FlagPath | FlagQuery)) - return LinkIsFragment; + // find out if it is link to itself then ignore it + if (!Compare(base, FlagPath | FlagQuery)) + return LinkIsFragment; - return LinkIsLocal; + return LinkIsLocal; } - /********************************************************/ - - size_t TUri::PrintSize(ui32 flags) const { - size_t len = 10; - flags &= FieldsSet; // can't output what we don't have - if (flags & FlagHostAscii) - flags &= ~FlagHost; // don't want to print both of them - ui32 opt = 1; - for (int fld = 0; opt <= flags && fld < FieldAllMAX; ++fld, opt <<= 1) { - if (opt & flags) { - const TStringBuf& v = Fields[fld]; - if (v.IsInited()) { - if (opt & FlagAuth) - len += 3 * v.length() + 1; - else - len += v.length() + 1; - } + /********************************************************/ + + size_t TUri::PrintSize(ui32 flags) const { + size_t len = 10; + flags &= FieldsSet; // can't output what we don't have + if (flags & FlagHostAscii) + flags &= ~FlagHost; // don't want to print both of them + ui32 opt = 1; + for (int fld = 0; opt <= flags && fld < FieldAllMAX; ++fld, opt <<= 1) { + if (opt & flags) { + const TStringBuf& v = Fields[fld]; + if (v.IsInited()) { + if (opt & FlagAuth) + len += 3 * v.length() + 1; + else + len += v.length() + 1; + } } } - - return len; + + return len; } - IOutputStream& TUri::PrintImpl(IOutputStream& out, int flags) const { - TStringBuf v; + IOutputStream& TUri::PrintImpl(IOutputStream& out, int flags) const { + TStringBuf v; - const int wantFlags = flags; // save the original - flags &= FieldsSet; // can't print what we don't have - if (flags & FlagHostAscii) - flags |= FlagHost; // to make host checks simpler below + const int wantFlags = flags; // save the original + flags &= FieldsSet; // can't print what we don't have + if (flags & FlagHostAscii) + flags |= FlagHost; // to make host checks simpler below - if (flags & FlagScheme) { - v = Fields[FieldScheme]; + if (flags & FlagScheme) { + v = Fields[FieldScheme]; if (!v.empty()) - out << v << ':'; - } - - TStringBuf host; - if (flags & FlagHost) { - const EField fldhost = - flags & FlagHostAscii ? FieldHostAscii : FieldHost; - host = Fields[fldhost]; - } - - TStringBuf port; - if ((flags & FlagPort) && 0 != Port && Port != DefaultPort) - port = Fields[FieldPort]; - - if (host) { - if (wantFlags & FlagScheme) - out << "//"; - - if (flags & FlagAuth) { - if (flags & FlagUser) { - v = Fields[FieldUser]; + out << v << ':'; + } + + TStringBuf host; + if (flags & FlagHost) { + const EField fldhost = + flags & FlagHostAscii ? FieldHostAscii : FieldHost; + host = Fields[fldhost]; + } + + TStringBuf port; + if ((flags & FlagPort) && 0 != Port && Port != DefaultPort) + port = Fields[FieldPort]; + + if (host) { + if (wantFlags & FlagScheme) + out << "//"; + + if (flags & FlagAuth) { + if (flags & FlagUser) { + v = Fields[FieldUser]; if (!v.empty()) - TEncoder::EncodeNotAlnum(out, v); - } - - if (flags & FlagPass) { - v = Fields[FieldPass]; - if (v.IsInited()) { - out << ':'; - TEncoder::EncodeAll(out, v); - } - } - - out << '@'; + TEncoder::EncodeNotAlnum(out, v); + } + + if (flags & FlagPass) { + v = Fields[FieldPass]; + if (v.IsInited()) { + out << ':'; + TEncoder::EncodeAll(out, v); + } + } + + out << '@'; } - out << host; + out << host; - if (port) - out << ':'; + if (port) + out << ':'; } - if (port) - out << port; + if (port) + out << port; - if (flags & FlagPath) { - v = Fields[FieldPath]; - // for relative, empty path is not the same as missing + if (flags & FlagPath) { + v = Fields[FieldPath]; + // for relative, empty path is not the same as missing if (v.empty() && 0 == (flags & FlagHost)) v = TStringBuf("."); - out << v; - } - - if (flags & FlagQuery) { - v = Fields[FieldQuery]; - if (v.IsInited()) - out << '?' << v; - } - - if (flags & FlagFrag) { - v = Fields[FieldFrag]; - if (v.IsInited()) - out << '#' << v; - } - - return out; + out << v; + } + + if (flags & FlagQuery) { + v = Fields[FieldQuery]; + if (v.IsInited()) + out << '?' << v; + } + + if (flags & FlagFrag) { + v = Fields[FieldFrag]; + if (v.IsInited()) + out << '#' << v; + } + + return out; } - /********************************************************/ - int TUri::CompareField(EField fld, const TUri& url) const { - const TStringBuf& v0 = GetField(fld); - const TStringBuf& v1 = url.GetField(fld); - switch (fld) { - case FieldScheme: - case FieldHost: - return CompareNoCase(v0, v1); - default: - return v0.compare(v1); - } + /********************************************************/ + int TUri::CompareField(EField fld, const TUri& url) const { + const TStringBuf& v0 = GetField(fld); + const TStringBuf& v1 = url.GetField(fld); + switch (fld) { + case FieldScheme: + case FieldHost: + return CompareNoCase(v0, v1); + default: + return v0.compare(v1); + } } - /********************************************************/ - int TUri::Compare(const TUri& url, int flags) const { - // first compare fields with default values - if (flags & FlagPort) { - const int ret = GetPort() - url.GetPort(); - if (ret) - return ret; - flags &= ~FlagPort; - } - - // compare remaining sets of available fields - const int rtflags = flags & url.FieldsSet; - flags &= FieldsSet; - const int fldcmp = flags - rtflags; - if (fldcmp) - return fldcmp; - - // field sets are the same, compare the fields themselves - for (int i = 0; i < FieldAllMAX; ++i) { - const EField fld = EField(i); - if (flags & FldFlag(fld)) { - const int ret = CompareField(fld, url); - if (ret) - return ret; - } - } - - return 0; + /********************************************************/ + int TUri::Compare(const TUri& url, int flags) const { + // first compare fields with default values + if (flags & FlagPort) { + const int ret = GetPort() - url.GetPort(); + if (ret) + return ret; + flags &= ~FlagPort; + } + + // compare remaining sets of available fields + const int rtflags = flags & url.FieldsSet; + flags &= FieldsSet; + const int fldcmp = flags - rtflags; + if (fldcmp) + return fldcmp; + + // field sets are the same, compare the fields themselves + for (int i = 0; i < FieldAllMAX; ++i) { + const EField fld = EField(i); + if (flags & FldFlag(fld)) { + const int ret = CompareField(fld, url); + if (ret) + return ret; + } + } + + return 0; } - /********************************************************/ - bool TUri::PathOperation(char*& pathPtr, char*& pathEnd, int correctAbs) { - if (!pathPtr) - return false; - if (pathPtr == pathEnd) - return true; + /********************************************************/ + bool TUri::PathOperation(char*& pathPtr, char*& pathEnd, int correctAbs) { + if (!pathPtr) + return false; + if (pathPtr == pathEnd) + return true; - if ((pathEnd - pathPtr) >= 2 && *(pathEnd - 2) == '/' && *(pathEnd - 1) == '.') { - --pathEnd; + if ((pathEnd - pathPtr) >= 2 && *(pathEnd - 2) == '/' && *(pathEnd - 1) == '.') { + --pathEnd; } - char* p_wr = pathEnd; - int upCount = 0; - - char* p_prev = pathEnd; - Y_ASSERT(p_prev > pathPtr); - while (p_prev > pathPtr && *(p_prev - 1) == '/') - p_prev--; - - for (char* p_rd = p_prev; p_rd; p_rd = p_prev) { - Y_ASSERT(p_rd == pathEnd || p_rd[0] == '/'); - p_prev = nullptr; - - char* p = p_rd; - - if (p > pathPtr) { - for (p--; *p != '/'; p--) { - if (p == pathPtr) - break; - } - if (*p == '/') { - p_prev = p++; - if ((p_prev - pathPtr >= 6 && !strnicmp(p_prev - 6, "http://", 7)) || - (p_prev - pathPtr >= 7 && !strnicmp(p_prev - 7, "https://", 8))) { - --p_prev; - --p; - } else { - //skip multiple from head '/' - while (p_prev > pathPtr && *(p_prev - 1) == '/') - p_prev--; - } - } - } - - Y_ASSERT(p_prev == nullptr || p_prev[0] == '/'); - //and the first symbol !='/' after p_prev is p - - if (p == p_rd) { - //empty block: - if (p_prev) { //either tail: - Y_ASSERT(p_rd == p_wr && *(p - 1) == '/'); - --p_wr; - continue; - } else { //or head of abs path - *(--p_wr) = '/'; + char* p_wr = pathEnd; + int upCount = 0; + + char* p_prev = pathEnd; + Y_ASSERT(p_prev > pathPtr); + while (p_prev > pathPtr && *(p_prev - 1) == '/') + p_prev--; + + for (char* p_rd = p_prev; p_rd; p_rd = p_prev) { + Y_ASSERT(p_rd == pathEnd || p_rd[0] == '/'); + p_prev = nullptr; + + char* p = p_rd; + + if (p > pathPtr) { + for (p--; *p != '/'; p--) { + if (p == pathPtr) + break; + } + if (*p == '/') { + p_prev = p++; + if ((p_prev - pathPtr >= 6 && !strnicmp(p_prev - 6, "http://", 7)) || + (p_prev - pathPtr >= 7 && !strnicmp(p_prev - 7, "https://", 8))) { + --p_prev; + --p; + } else { + //skip multiple from head '/' + while (p_prev > pathPtr && *(p_prev - 1) == '/') + p_prev--; + } + } + } + + Y_ASSERT(p_prev == nullptr || p_prev[0] == '/'); + //and the first symbol !='/' after p_prev is p + + if (p == p_rd) { + //empty block: + if (p_prev) { //either tail: + Y_ASSERT(p_rd == p_wr && *(p - 1) == '/'); + --p_wr; + continue; + } else { //or head of abs path + *(--p_wr) = '/'; break; - } + } } - - if (p[0] == '.') { - if (p + 1 == p_rd) { - if (correctAbs || p_prev > pathPtr || pathPtr[0] != '/') - // ignore "./" - continue; - } else { - if ((p[1] == '.') && (p + 2 == p_rd)) { - // register "../" but not print - upCount++; - continue; - } + + if (p[0] == '.') { + if (p + 1 == p_rd) { + if (correctAbs || p_prev > pathPtr || pathPtr[0] != '/') + // ignore "./" + continue; + } else { + if ((p[1] == '.') && (p + 2 == p_rd)) { + // register "../" but not print + upCount++; + continue; + } } } - if (upCount) { - //unregister "../" and not print - upCount--; + if (upCount) { + //unregister "../" and not print + upCount--; continue; } - // print - Y_ASSERT(p < p_rd); - Y_ASSERT(!p_prev || *(p - 1) == '/'); - if (p_wr == p_rd) { //just skip - p_wr = p; - } else { //copy - int l = p_rd - p + 1; - p_wr -= l; - memmove(p_wr, p, l); + // print + Y_ASSERT(p < p_rd); + Y_ASSERT(!p_prev || *(p - 1) == '/'); + if (p_wr == p_rd) { //just skip + p_wr = p; + } else { //copy + int l = p_rd - p + 1; + p_wr -= l; + memmove(p_wr, p, l); } } - if (upCount) { - if (*pathPtr != '/') { - if (pathEnd == p_wr && *(p_wr - 1) == '.') { - Y_ASSERT(*(p_wr - 2) == '.'); - p_wr -= 2; - upCount--; - } - for (; upCount > 0; upCount--) { - *(--p_wr) = '/'; + if (upCount) { + if (*pathPtr != '/') { + if (pathEnd == p_wr && *(p_wr - 1) == '.') { + Y_ASSERT(*(p_wr - 2) == '.'); + p_wr -= 2; + upCount--; + } + for (; upCount > 0; upCount--) { + *(--p_wr) = '/'; *(--p_wr) = '.'; *(--p_wr) = '.'; } - } else { - if (correctAbs > 0) - return false; - if (correctAbs == 0) { - //Bad path but present in RFC: - // "Similarly, parsers must avoid treating "." and ".." - // as special when they are not complete components of - // a relative path. " - for (; upCount > 0; upCount--) { - *(--p_wr) = '.'; - *(--p_wr) = '.'; - *(--p_wr) = '/'; - } - } else { - upCount = false; - } + } else { + if (correctAbs > 0) + return false; + if (correctAbs == 0) { + //Bad path but present in RFC: + // "Similarly, parsers must avoid treating "." and ".." + // as special when they are not complete components of + // a relative path. " + for (; upCount > 0; upCount--) { + *(--p_wr) = '.'; + *(--p_wr) = '.'; + *(--p_wr) = '/'; + } + } else { + upCount = false; + } } } - Y_ASSERT(p_wr >= pathPtr); - - if (upCount) - return false; - pathPtr = p_wr; - return true; - } - - /********************************************************/ - const char* LinkTypeToString(const TUri::TLinkType& t) { - switch (t) { - case TUri::LinkIsBad: - return "LinkIsBad"; - case TUri::LinkBadAbs: - return "LinkBadAbs"; - case TUri::LinkIsFragment: - return "LinkIsFragment"; - case TUri::LinkIsLocal: - return "LinkIsLocal"; - case TUri::LinkIsGlobal: - return "LinkIsGlobal"; - } - Y_ASSERT(0); - return ""; + Y_ASSERT(p_wr >= pathPtr); + + if (upCount) + return false; + pathPtr = p_wr; + return true; + } + + /********************************************************/ + const char* LinkTypeToString(const TUri::TLinkType& t) { + switch (t) { + case TUri::LinkIsBad: + return "LinkIsBad"; + case TUri::LinkBadAbs: + return "LinkBadAbs"; + case TUri::LinkIsFragment: + return "LinkIsFragment"; + case TUri::LinkIsLocal: + return "LinkIsLocal"; + case TUri::LinkIsGlobal: + return "LinkIsGlobal"; + } + Y_ASSERT(0); + return ""; } } diff --git a/library/cpp/uri/uri.h b/library/cpp/uri/uri.h index 3b6c19fe4a..05908c4e96 100644 --- a/library/cpp/uri/uri.h +++ b/library/cpp/uri/uri.h @@ -2,625 +2,625 @@ #include "common.h" #include "encode.h" - + #include <library/cpp/charset/doccodes.h> #include <util/generic/buffer.h> #include <util/generic/ptr.h> #include <util/generic/singleton.h> #include <util/generic/string.h> #include <util/memory/alloc.h> -#include <util/stream/mem.h> +#include <util/stream/mem.h> #include <util/stream/output.h> -#include <util/stream/str.h> -#include <util/system/yassert.h> +#include <util/stream/str.h> +#include <util/system/yassert.h> #include <cstdlib> namespace NUri { - /********************************************************/ - class TUri - : public TFeature, - public TField, - public TScheme, - public TState { - public: - enum TLinkType { - LinkIsBad, - LinkBadAbs, - LinkIsFragment, - LinkIsLocal, - LinkIsGlobal - }; - - private: + /********************************************************/ + class TUri + : public TFeature, + public TField, + public TScheme, + public TState { + public: + enum TLinkType { + LinkIsBad, + LinkBadAbs, + LinkIsFragment, + LinkIsLocal, + LinkIsGlobal + }; + + private: TBuffer Buffer; - TStringBuf Fields[FieldAllMAX]; - ui32 FieldsSet; - ui16 Port; - ui16 DefaultPort; - TScheme::EKind Scheme; - /// contains fields out of buffer (and possibly not null-terminated) - ui32 FieldsDirty; - - private: - void Alloc(size_t len) { - Dealloc(); // to prevent copy below + TStringBuf Fields[FieldAllMAX]; + ui32 FieldsSet; + ui16 Port; + ui16 DefaultPort; + TScheme::EKind Scheme; + /// contains fields out of buffer (and possibly not null-terminated) + ui32 FieldsDirty; + + private: + void Alloc(size_t len) { + Dealloc(); // to prevent copy below Buffer.Resize(len); - } - void Dealloc() { + } + void Dealloc() { Buffer.Clear(); - } - - void ClearImpl() { - Port = 0; - FieldsSet = 0; - Scheme = SchemeEmpty; - FieldsDirty = 0; - } - - void CopyData(const TUri& url) { - FieldsSet = url.FieldsSet; - Port = url.Port; - DefaultPort = url.DefaultPort; - Scheme = url.Scheme; - FieldsDirty = url.FieldsDirty; - } - - void CopyImpl(const TUri& url) { - for (int i = 0; i < FieldAllMAX; ++i) - Fields[i] = url.Fields[i]; + } + + void ClearImpl() { + Port = 0; + FieldsSet = 0; + Scheme = SchemeEmpty; + FieldsDirty = 0; + } + + void CopyData(const TUri& url) { + FieldsSet = url.FieldsSet; + Port = url.Port; + DefaultPort = url.DefaultPort; + Scheme = url.Scheme; + FieldsDirty = url.FieldsDirty; + } + + void CopyImpl(const TUri& url) { + for (int i = 0; i < FieldAllMAX; ++i) + Fields[i] = url.Fields[i]; RewriteImpl(); - } - - private: - static ui32 FldFlag(EField fld) { - return 1 << fld; - } - - public: - static bool FldIsValid(EField fld) { - return 0 <= fld && FieldAllMAX > fld; - } - - bool FldSetCmp(ui32 chk, ui32 exp) const { - return (FieldsSet & chk) == exp; - } - - bool FldSetCmp(ui32 chk) const { - return FldSetCmp(chk, chk); - } - - bool FldIsSet(EField fld) const { - return !FldSetCmp(FldFlag(fld), 0); - } - - private: - void FldMarkSet(EField fld) { - FieldsSet |= FldFlag(fld); - } - - void FldMarkUnset(EField fld) { - FieldsSet &= ~FldFlag(fld); - } - - // use when we know the field is dirty or RewriteImpl will be called - void FldSetNoDirty(EField fld, const TStringBuf& value) { - Fields[fld] = value; - FldMarkSet(fld); - } - - void FldSet(EField fld, const TStringBuf& value) { - FldSetNoDirty(fld, value); - FldMarkDirty(fld); - } - - const TStringBuf& FldGet(EField fld) const { - return Fields[fld]; - } - - private: - /// depending on value, clears or sets it - void FldChkSet(EField fld, const TStringBuf& value) { - if (value.IsInited()) - FldSet(fld, value); - else - FldClr(fld); - } - void FldChkSet(EField fld, const TUri& other) { - FldChkSet(fld, other.GetField(fld)); - } - - /// set only if initialized - bool FldTrySet(EField fld, const TStringBuf& value) { - const bool ok = value.IsInited(); - if (ok) - FldSet(fld, value); - return ok; - } - bool FldTrySet(EField fld, const TUri& other) { - return FldTrySet(fld, other.GetField(fld)); - } - - private: - /// copies the value if it fits - bool FldTryCpy(EField fld, const TStringBuf& value); - - // main method: sets the field value, possibly copies, etc. - bool FldSetImpl(EField fld, TStringBuf value, bool strconst = false, bool nocopy = false); - - public: // clear a field - void FldClr(EField fld) { - Fields[fld].Clear(); - FldMarkUnset(fld); - FldMarkClean(fld); - } - - bool FldTryClr(EField field) { - const bool ok = FldIsSet(field); - if (ok) - FldClr(field); - return ok; - } - - public: // set a field value: might leave state dirty and require a Rewrite() - // copies if fits and not dirty, sets and marks dirty otherwise - bool FldMemCpy(EField field, const TStringBuf& value) { - return FldSetImpl(field, value, false); - } - - // uses directly, marks dirty - /// @note client MUST guarantee value will be alive until Rewrite is called - bool FldMemSet(EField field, const TStringBuf& value) { - return FldSetImpl(field, value, false, true); - } - - // uses directly, doesn't mark dirty (value scope exceeds "this") - bool FldMemUse(EField field, const TStringBuf& value) { - return FldSetImpl(field, value, true); - } - - // uses directly, doesn't mark dirty - template <size_t size> - bool FldMemSet(EField field, const char (&value)[size]) { + } + + private: + static ui32 FldFlag(EField fld) { + return 1 << fld; + } + + public: + static bool FldIsValid(EField fld) { + return 0 <= fld && FieldAllMAX > fld; + } + + bool FldSetCmp(ui32 chk, ui32 exp) const { + return (FieldsSet & chk) == exp; + } + + bool FldSetCmp(ui32 chk) const { + return FldSetCmp(chk, chk); + } + + bool FldIsSet(EField fld) const { + return !FldSetCmp(FldFlag(fld), 0); + } + + private: + void FldMarkSet(EField fld) { + FieldsSet |= FldFlag(fld); + } + + void FldMarkUnset(EField fld) { + FieldsSet &= ~FldFlag(fld); + } + + // use when we know the field is dirty or RewriteImpl will be called + void FldSetNoDirty(EField fld, const TStringBuf& value) { + Fields[fld] = value; + FldMarkSet(fld); + } + + void FldSet(EField fld, const TStringBuf& value) { + FldSetNoDirty(fld, value); + FldMarkDirty(fld); + } + + const TStringBuf& FldGet(EField fld) const { + return Fields[fld]; + } + + private: + /// depending on value, clears or sets it + void FldChkSet(EField fld, const TStringBuf& value) { + if (value.IsInited()) + FldSet(fld, value); + else + FldClr(fld); + } + void FldChkSet(EField fld, const TUri& other) { + FldChkSet(fld, other.GetField(fld)); + } + + /// set only if initialized + bool FldTrySet(EField fld, const TStringBuf& value) { + const bool ok = value.IsInited(); + if (ok) + FldSet(fld, value); + return ok; + } + bool FldTrySet(EField fld, const TUri& other) { + return FldTrySet(fld, other.GetField(fld)); + } + + private: + /// copies the value if it fits + bool FldTryCpy(EField fld, const TStringBuf& value); + + // main method: sets the field value, possibly copies, etc. + bool FldSetImpl(EField fld, TStringBuf value, bool strconst = false, bool nocopy = false); + + public: // clear a field + void FldClr(EField fld) { + Fields[fld].Clear(); + FldMarkUnset(fld); + FldMarkClean(fld); + } + + bool FldTryClr(EField field) { + const bool ok = FldIsSet(field); + if (ok) + FldClr(field); + return ok; + } + + public: // set a field value: might leave state dirty and require a Rewrite() + // copies if fits and not dirty, sets and marks dirty otherwise + bool FldMemCpy(EField field, const TStringBuf& value) { + return FldSetImpl(field, value, false); + } + + // uses directly, marks dirty + /// @note client MUST guarantee value will be alive until Rewrite is called + bool FldMemSet(EField field, const TStringBuf& value) { + return FldSetImpl(field, value, false, true); + } + + // uses directly, doesn't mark dirty (value scope exceeds "this") + bool FldMemUse(EField field, const TStringBuf& value) { + return FldSetImpl(field, value, true); + } + + // uses directly, doesn't mark dirty + template <size_t size> + bool FldMemSet(EField field, const char (&value)[size]) { static_assert(size > 0); return FldSetImpl(field, TStringBuf(value, size - 1), true); - } - - // duplicate one field to another - bool FldDup(EField src, EField dst) { - if (!FldIsSet(src) || !FldIsValid(dst)) - return false; - FldSetNoDirty(dst, FldGet(src)); - if (FldIsDirty(src)) - FldMarkDirty(dst); - else - FldMarkClean(dst); - return true; - } - - // move one field to another - bool FldMov(EField src, EField dst) { - if (!FldDup(src, dst)) - return false; - FldClr(src); - return true; - } - - private: - bool IsInBuffer(const char* buf) const { + } + + // duplicate one field to another + bool FldDup(EField src, EField dst) { + if (!FldIsSet(src) || !FldIsValid(dst)) + return false; + FldSetNoDirty(dst, FldGet(src)); + if (FldIsDirty(src)) + FldMarkDirty(dst); + else + FldMarkClean(dst); + return true; + } + + // move one field to another + bool FldMov(EField src, EField dst) { + if (!FldDup(src, dst)) + return false; + FldClr(src); + return true; + } + + private: + bool IsInBuffer(const char* buf) const { return buf >= Buffer.data() && buf < Buffer.data() + Buffer.size(); - } + } - public: - bool FldIsDirty() const { - return 0 != FieldsDirty; - } + public: + bool FldIsDirty() const { + return 0 != FieldsDirty; + } - bool FldIsDirty(EField fld) const { - return 0 != (FieldsDirty & FldFlag(fld)); - } + bool FldIsDirty(EField fld) const { + return 0 != (FieldsDirty & FldFlag(fld)); + } - private: - void FldMarkDirty(EField fld) { - FieldsDirty |= FldFlag(fld); - } + private: + void FldMarkDirty(EField fld) { + FieldsDirty |= FldFlag(fld); + } - void FldMarkClean(EField fld) { - FieldsDirty &= ~FldFlag(fld); - } + void FldMarkClean(EField fld) { + FieldsDirty &= ~FldFlag(fld); + } - void RewriteImpl(); + void RewriteImpl(); - public: - static TState::EParsed CheckHost(const TStringBuf& host); + public: + static TState::EParsed CheckHost(const TStringBuf& host); - // convert a [potential] IDN to ascii - static TMallocPtr<char> IDNToAscii(const wchar32* idna); - static TMallocPtr<char> IDNToAscii(const TStringBuf& host, ECharset enc = CODES_UTF8); + // convert a [potential] IDN to ascii + static TMallocPtr<char> IDNToAscii(const wchar32* idna); + static TMallocPtr<char> IDNToAscii(const TStringBuf& host, ECharset enc = CODES_UTF8); - // convert hosts with percent-encoded or extended chars + // convert hosts with percent-encoded or extended chars - // returns non-empty string if host can be converted to ASCII with given parameters - static TStringBuf HostToAscii(TStringBuf host, TMallocPtr<char>& buf, bool hasExtended, bool allowIDN, ECharset enc = CODES_UTF8); + // returns non-empty string if host can be converted to ASCII with given parameters + static TStringBuf HostToAscii(TStringBuf host, TMallocPtr<char>& buf, bool hasExtended, bool allowIDN, ECharset enc = CODES_UTF8); - // returns host if already ascii, or non-empty if it can be converted - static TStringBuf HostToAscii(const TStringBuf& host, TMallocPtr<char>& buf, bool allowIDN, ECharset enc = CODES_UTF8); + // returns host if already ascii, or non-empty if it can be converted + static TStringBuf HostToAscii(const TStringBuf& host, TMallocPtr<char>& buf, bool allowIDN, ECharset enc = CODES_UTF8); - public: - explicit TUri(unsigned defaultPort = 0) - : FieldsSet(0) - , Port(0) - , DefaultPort(static_cast<ui16>(defaultPort)) - , Scheme(SchemeEmpty) - , FieldsDirty(0) - { - } + public: + explicit TUri(unsigned defaultPort = 0) + : FieldsSet(0) + , Port(0) + , DefaultPort(static_cast<ui16>(defaultPort)) + , Scheme(SchemeEmpty) + , FieldsDirty(0) + { + } - TUri(const TStringBuf& host, ui16 port, const TStringBuf& path, const TStringBuf& query = TStringBuf(), const TStringBuf& scheme = "http", unsigned defaultPort = 0); + TUri(const TStringBuf& host, ui16 port, const TStringBuf& path, const TStringBuf& query = TStringBuf(), const TStringBuf& scheme = "http", unsigned defaultPort = 0); - TUri(const TUri& url) + TUri(const TUri& url) : FieldsSet(url.FieldsSet) - , Port(url.Port) - , DefaultPort(url.DefaultPort) - , Scheme(url.Scheme) - , FieldsDirty(url.FieldsDirty) - { - CopyImpl(url); - } - - ~TUri() { - Clear(); - } - - void Copy(const TUri& url) { - if (&url != this) { - CopyData(url); - CopyImpl(url); - } - } - - void Clear() { - Dealloc(); - ClearImpl(); - } - - ui32 GetFieldMask() const { - return FieldsSet; - } - - ui32 GetUrlFieldMask() const { - return GetFieldMask() & FlagUrlFields; - } - - ui32 GetDirtyMask() const { - return FieldsDirty; - } - - void CheckMissingFields(); - - // Process methods - - void Rewrite() { - if (FldIsDirty()) - RewriteImpl(); - } - - private: - TState::EParsed AssignImpl(const TParser& parser, TScheme::EKind defscheme = SchemeEmpty); - - TState::EParsed ParseImpl(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, TScheme::EKind defscheme = SchemeEmpty, ECharset enc = CODES_UTF8); - - public: - TState::EParsed Assign(const TParser& parser, TScheme::EKind defscheme = SchemeEmpty) { - const TState::EParsed ret = AssignImpl(parser, defscheme); - if (ParsedOK == ret) - Rewrite(); - return ret; - } - - TState::EParsed ParseUri(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, ECharset enc = CODES_UTF8) { - const TState::EParsed ret = ParseImpl(url, flags, maxlen, SchemeEmpty, enc); - if (ParsedOK == ret) - Rewrite(); - return ret; - } - - // parses absolute URIs - // prepends default scheme (unless unknown) if URI has none - TState::EParsed ParseAbsUri(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, TScheme::EKind defscheme = SchemeUnknown, ECharset enc = CODES_UTF8); - - TState::EParsed ParseAbsOrHttpUri(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, ECharset enc = CODES_UTF8) { - return ParseAbsUri(url, flags, maxlen, SchemeHTTP, enc); - } - - TState::EParsed Parse(const TStringBuf& url, const TUri& base, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, ECharset enc = CODES_UTF8); - - TState::EParsed Parse(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault) { - return ParseUri(url, flags); - } - - TState::EParsed Parse(const TStringBuf& url, const TParseFlags& flags, const TStringBuf& base_url, ui32 maxlen = 0, ECharset enc = CODES_UTF8); - - TState::EParsed ParseAbs(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, const TStringBuf& base_url = TStringBuf(), ui32 maxlen = 0, ECharset enc = CODES_UTF8) { - const TState::EParsed result = Parse(url, flags, base_url, maxlen, enc); - return ParsedOK != result || IsValidGlobal() ? result : ParsedBadFormat; - } - - // correctAbs works with head "/.." portions: - // 1 - reject URL - // 0 - keep portions - // -1 - ignore portions - - void Merge(const TUri& base, int correctAbs = -1); + , Port(url.Port) + , DefaultPort(url.DefaultPort) + , Scheme(url.Scheme) + , FieldsDirty(url.FieldsDirty) + { + CopyImpl(url); + } - TLinkType Normalize(const TUri& base, const TStringBuf& link, const TStringBuf& codebase = TStringBuf(), long careFlags = FeaturesDefault, ECharset enc = CODES_UTF8); + ~TUri() { + Clear(); + } + + void Copy(const TUri& url) { + if (&url != this) { + CopyData(url); + CopyImpl(url); + } + } - private: - int PrintFlags(int flags) const { - if (0 == (FlagUrlFields & flags)) - flags |= FlagUrlFields; - return flags; + void Clear() { + Dealloc(); + ClearImpl(); } - - protected: - size_t PrintSize(ui32 flags) const; - - // Output method, prints to stream - IOutputStream& PrintImpl(IOutputStream& out, int flags) const; - - char* PrintImpl(char* str, size_t size, int flags) const { - TMemoryOutput out(str, size); - PrintImpl(out, flags) << '\0'; - return str; - } - - static bool IsAbsPath(const TStringBuf& path) { - return 1 <= path.length() && path[0] == '/'; - } - - bool IsAbsPathImpl() const { - return IsAbsPath(GetField(FieldPath)); - } - - public: - // Output method, prints to stream - IOutputStream& Print(IOutputStream& out, int flags = FlagUrlFields) const { - return PrintImpl(out, PrintFlags(flags)); - } - - // Output method, print to str, allocate memory if str is NULL - // Should be deprecated - char* Print(char* str, size_t size, int flags = FlagUrlFields) const { - return nullptr == str ? Serialize(flags) : Serialize(str, size, flags); - } - - char* Serialize(char* str, size_t size, int flags = FlagUrlFields) const { - Y_ASSERT(str); - flags = PrintFlags(flags); - const size_t printSize = PrintSize(flags) + 1; - return printSize > size ? nullptr : PrintImpl(str, size, flags); - } - - char* Serialize(int flags = FlagUrlFields) const { - flags = PrintFlags(flags); - const size_t size = PrintSize(flags) + 1; - return PrintImpl(static_cast<char*>(malloc(size)), size, flags); - } - - // Output method to str - void Print(TString& str, int flags = FlagUrlFields) const { - flags = PrintFlags(flags); - str.reserve(str.length() + PrintSize(flags)); - TStringOutput out(str); - PrintImpl(out, flags); - } - - TString PrintS(int flags = FlagUrlFields) const { - TString str; - Print(str, flags); - return str; - } - - // Only non-default scheme and port are printed - char* PrintHost(char* str, size_t size) const { - return Print(str, size, (Scheme != SchemeHTTP ? FlagScheme : 0) | FlagHostPort); - } - TString PrintHostS() const { - return PrintS((Scheme != SchemeHTTP ? FlagScheme : 0) | FlagHostPort); - } - - // Info methods - int Compare(const TUri& A, int flags = FlagUrlFields) const; - - int CompareField(EField fld, const TUri& url) const; - - const TStringBuf& GetField(EField fld) const { - return FldIsValid(fld) && FldIsSet(fld) ? FldGet(fld) : Default<TStringBuf>(); - } - - ui16 GetPort() const { - return 0 == Port ? DefaultPort : Port; - } - - const TStringBuf& GetHost() const { - if (GetFieldMask() & FlagHostAscii) - return FldGet(FieldHostAscii); - if (GetFieldMask() & FlagHost) - return FldGet(FieldHost); - return Default<TStringBuf>(); - } - - bool UseHostAscii() { - return FldMov(FieldHostAscii, FieldHost); - } - - TScheme::EKind GetScheme() const { - return Scheme; - } - const TSchemeInfo& GetSchemeInfo() const { - return TSchemeInfo::Get(Scheme); - } - - bool IsNull(ui32 flags = FlagScheme | FlagHost | FlagPath) const { - return !FldSetCmp(flags); - } - - bool IsNull(EField fld) const { - return !FldIsSet(fld); - } - - bool IsValidAbs() const { - if (IsNull(FlagScheme | FlagHost | FlagPath)) - return false; - return IsAbsPathImpl(); - } - - bool IsValidGlobal() const { - if (IsNull(FlagScheme | FlagHost)) - return false; - if (IsNull(FlagPath)) - return true; - return IsAbsPathImpl(); - } - - bool IsRootless() const { - return FldSetCmp(FlagScheme | FlagHost | FlagPath, FlagScheme | FlagPath) && !IsAbsPathImpl(); - } - - // for RFC 2396 compatibility - bool IsOpaque() const { - return IsRootless(); - } - - // Inline helpers - TUri& operator=(const TUri& u) { - Copy(u); - return *this; - } - - bool operator!() const { - return IsNull(); - } - - bool Equal(const TUri& A, int flags = FlagUrlFields) const { - return (Compare(A, flags) == 0); - } - - bool Less(const TUri& A, int flags = FlagUrlFields) const { - return (Compare(A, flags) < 0); - } - - bool operator==(const TUri& A) const { - return Equal(A, FlagNoFrag); - } - - bool operator!=(const TUri& A) const { - return !Equal(A, FlagNoFrag); - } - - bool operator<(const TUri& A) const { - return Less(A, FlagNoFrag); - } - - bool IsSameDocument(const TUri& other) const { - // pre: both *this and 'other' should be normalized to valid abs - Y_ASSERT(IsValidAbs()); - return Equal(other, FlagNoFrag); - } - - bool IsLocal(const TUri& other) const { - // pre: both *this and 'other' should be normalized to valid abs - Y_ASSERT(IsValidAbs() && other.IsValidAbs()); - return Equal(other, FlagScheme | FlagHostPort); - } - - TLinkType Locality(const TUri& other) const { - if (IsSameDocument(other)) - return LinkIsFragment; - else if (IsLocal(other)) - return LinkIsLocal; - return LinkIsGlobal; - } - - static IOutputStream& ReEncodeField(IOutputStream& out, const TStringBuf& val, EField fld, long flags = FeaturesEncodeDecode) { - return NEncode::TEncoder::ReEncode(out, val, NEncode::TEncodeMapper(flags, fld)); - } - - static IOutputStream& ReEncodeToField(IOutputStream& out, const TStringBuf& val, EField srcfld, long srcflags, EField dstfld, long dstflags) { - return NEncode::TEncoder::ReEncodeTo(out, val, NEncode::TEncodeMapper(srcflags, srcfld), NEncode::TEncodeToMapper(dstflags, dstfld)); - } - - static IOutputStream& ReEncode(IOutputStream& out, const TStringBuf& val, long flags = FeaturesEncodeDecode) { - return ReEncodeField(out, val, FieldAllMAX, flags); - } - - static int PathOperationFlag(const TParseFlags& flags) { - return flags & FeaturePathDenyRootParent ? 1 - : flags & FeaturePathStripRootParent ? -1 : 0; - } - - static bool PathOperation(char*& pathBeg, char*& pathEnd, int correctAbs); - - private: - const TSchemeInfo& SetSchemeImpl(const TSchemeInfo& info) { - Scheme = info.Kind; - DefaultPort = info.Port; + + ui32 GetFieldMask() const { + return FieldsSet; + } + + ui32 GetUrlFieldMask() const { + return GetFieldMask() & FlagUrlFields; + } + + ui32 GetDirtyMask() const { + return FieldsDirty; + } + + void CheckMissingFields(); + + // Process methods + + void Rewrite() { + if (FldIsDirty()) + RewriteImpl(); + } + + private: + TState::EParsed AssignImpl(const TParser& parser, TScheme::EKind defscheme = SchemeEmpty); + + TState::EParsed ParseImpl(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, TScheme::EKind defscheme = SchemeEmpty, ECharset enc = CODES_UTF8); + + public: + TState::EParsed Assign(const TParser& parser, TScheme::EKind defscheme = SchemeEmpty) { + const TState::EParsed ret = AssignImpl(parser, defscheme); + if (ParsedOK == ret) + Rewrite(); + return ret; + } + + TState::EParsed ParseUri(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, ECharset enc = CODES_UTF8) { + const TState::EParsed ret = ParseImpl(url, flags, maxlen, SchemeEmpty, enc); + if (ParsedOK == ret) + Rewrite(); + return ret; + } + + // parses absolute URIs + // prepends default scheme (unless unknown) if URI has none + TState::EParsed ParseAbsUri(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, TScheme::EKind defscheme = SchemeUnknown, ECharset enc = CODES_UTF8); + + TState::EParsed ParseAbsOrHttpUri(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, ECharset enc = CODES_UTF8) { + return ParseAbsUri(url, flags, maxlen, SchemeHTTP, enc); + } + + TState::EParsed Parse(const TStringBuf& url, const TUri& base, const TParseFlags& flags = FeaturesDefault, ui32 maxlen = 0, ECharset enc = CODES_UTF8); + + TState::EParsed Parse(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault) { + return ParseUri(url, flags); + } + + TState::EParsed Parse(const TStringBuf& url, const TParseFlags& flags, const TStringBuf& base_url, ui32 maxlen = 0, ECharset enc = CODES_UTF8); + + TState::EParsed ParseAbs(const TStringBuf& url, const TParseFlags& flags = FeaturesDefault, const TStringBuf& base_url = TStringBuf(), ui32 maxlen = 0, ECharset enc = CODES_UTF8) { + const TState::EParsed result = Parse(url, flags, base_url, maxlen, enc); + return ParsedOK != result || IsValidGlobal() ? result : ParsedBadFormat; + } + + // correctAbs works with head "/.." portions: + // 1 - reject URL + // 0 - keep portions + // -1 - ignore portions + + void Merge(const TUri& base, int correctAbs = -1); + + TLinkType Normalize(const TUri& base, const TStringBuf& link, const TStringBuf& codebase = TStringBuf(), long careFlags = FeaturesDefault, ECharset enc = CODES_UTF8); + + private: + int PrintFlags(int flags) const { + if (0 == (FlagUrlFields & flags)) + flags |= FlagUrlFields; + return flags; + } + + protected: + size_t PrintSize(ui32 flags) const; + + // Output method, prints to stream + IOutputStream& PrintImpl(IOutputStream& out, int flags) const; + + char* PrintImpl(char* str, size_t size, int flags) const { + TMemoryOutput out(str, size); + PrintImpl(out, flags) << '\0'; + return str; + } + + static bool IsAbsPath(const TStringBuf& path) { + return 1 <= path.length() && path[0] == '/'; + } + + bool IsAbsPathImpl() const { + return IsAbsPath(GetField(FieldPath)); + } + + public: + // Output method, prints to stream + IOutputStream& Print(IOutputStream& out, int flags = FlagUrlFields) const { + return PrintImpl(out, PrintFlags(flags)); + } + + // Output method, print to str, allocate memory if str is NULL + // Should be deprecated + char* Print(char* str, size_t size, int flags = FlagUrlFields) const { + return nullptr == str ? Serialize(flags) : Serialize(str, size, flags); + } + + char* Serialize(char* str, size_t size, int flags = FlagUrlFields) const { + Y_ASSERT(str); + flags = PrintFlags(flags); + const size_t printSize = PrintSize(flags) + 1; + return printSize > size ? nullptr : PrintImpl(str, size, flags); + } + + char* Serialize(int flags = FlagUrlFields) const { + flags = PrintFlags(flags); + const size_t size = PrintSize(flags) + 1; + return PrintImpl(static_cast<char*>(malloc(size)), size, flags); + } + + // Output method to str + void Print(TString& str, int flags = FlagUrlFields) const { + flags = PrintFlags(flags); + str.reserve(str.length() + PrintSize(flags)); + TStringOutput out(str); + PrintImpl(out, flags); + } + + TString PrintS(int flags = FlagUrlFields) const { + TString str; + Print(str, flags); + return str; + } + + // Only non-default scheme and port are printed + char* PrintHost(char* str, size_t size) const { + return Print(str, size, (Scheme != SchemeHTTP ? FlagScheme : 0) | FlagHostPort); + } + TString PrintHostS() const { + return PrintS((Scheme != SchemeHTTP ? FlagScheme : 0) | FlagHostPort); + } + + // Info methods + int Compare(const TUri& A, int flags = FlagUrlFields) const; + + int CompareField(EField fld, const TUri& url) const; + + const TStringBuf& GetField(EField fld) const { + return FldIsValid(fld) && FldIsSet(fld) ? FldGet(fld) : Default<TStringBuf>(); + } + + ui16 GetPort() const { + return 0 == Port ? DefaultPort : Port; + } + + const TStringBuf& GetHost() const { + if (GetFieldMask() & FlagHostAscii) + return FldGet(FieldHostAscii); + if (GetFieldMask() & FlagHost) + return FldGet(FieldHost); + return Default<TStringBuf>(); + } + + bool UseHostAscii() { + return FldMov(FieldHostAscii, FieldHost); + } + + TScheme::EKind GetScheme() const { + return Scheme; + } + const TSchemeInfo& GetSchemeInfo() const { + return TSchemeInfo::Get(Scheme); + } + + bool IsNull(ui32 flags = FlagScheme | FlagHost | FlagPath) const { + return !FldSetCmp(flags); + } + + bool IsNull(EField fld) const { + return !FldIsSet(fld); + } + + bool IsValidAbs() const { + if (IsNull(FlagScheme | FlagHost | FlagPath)) + return false; + return IsAbsPathImpl(); + } + + bool IsValidGlobal() const { + if (IsNull(FlagScheme | FlagHost)) + return false; + if (IsNull(FlagPath)) + return true; + return IsAbsPathImpl(); + } + + bool IsRootless() const { + return FldSetCmp(FlagScheme | FlagHost | FlagPath, FlagScheme | FlagPath) && !IsAbsPathImpl(); + } + + // for RFC 2396 compatibility + bool IsOpaque() const { + return IsRootless(); + } + + // Inline helpers + TUri& operator=(const TUri& u) { + Copy(u); + return *this; + } + + bool operator!() const { + return IsNull(); + } + + bool Equal(const TUri& A, int flags = FlagUrlFields) const { + return (Compare(A, flags) == 0); + } + + bool Less(const TUri& A, int flags = FlagUrlFields) const { + return (Compare(A, flags) < 0); + } + + bool operator==(const TUri& A) const { + return Equal(A, FlagNoFrag); + } + + bool operator!=(const TUri& A) const { + return !Equal(A, FlagNoFrag); + } + + bool operator<(const TUri& A) const { + return Less(A, FlagNoFrag); + } + + bool IsSameDocument(const TUri& other) const { + // pre: both *this and 'other' should be normalized to valid abs + Y_ASSERT(IsValidAbs()); + return Equal(other, FlagNoFrag); + } + + bool IsLocal(const TUri& other) const { + // pre: both *this and 'other' should be normalized to valid abs + Y_ASSERT(IsValidAbs() && other.IsValidAbs()); + return Equal(other, FlagScheme | FlagHostPort); + } + + TLinkType Locality(const TUri& other) const { + if (IsSameDocument(other)) + return LinkIsFragment; + else if (IsLocal(other)) + return LinkIsLocal; + return LinkIsGlobal; + } + + static IOutputStream& ReEncodeField(IOutputStream& out, const TStringBuf& val, EField fld, long flags = FeaturesEncodeDecode) { + return NEncode::TEncoder::ReEncode(out, val, NEncode::TEncodeMapper(flags, fld)); + } + + static IOutputStream& ReEncodeToField(IOutputStream& out, const TStringBuf& val, EField srcfld, long srcflags, EField dstfld, long dstflags) { + return NEncode::TEncoder::ReEncodeTo(out, val, NEncode::TEncodeMapper(srcflags, srcfld), NEncode::TEncodeToMapper(dstflags, dstfld)); + } + + static IOutputStream& ReEncode(IOutputStream& out, const TStringBuf& val, long flags = FeaturesEncodeDecode) { + return ReEncodeField(out, val, FieldAllMAX, flags); + } + + static int PathOperationFlag(const TParseFlags& flags) { + return flags & FeaturePathDenyRootParent ? 1 + : flags & FeaturePathStripRootParent ? -1 : 0; + } + + static bool PathOperation(char*& pathBeg, char*& pathEnd, int correctAbs); + + private: + const TSchemeInfo& SetSchemeImpl(const TSchemeInfo& info) { + Scheme = info.Kind; + DefaultPort = info.Port; if (!info.Str.empty()) - FldSetNoDirty(FieldScheme, info.Str); - return info; - } - const TSchemeInfo& SetSchemeImpl(TScheme::EKind scheme) { - return SetSchemeImpl(TSchemeInfo::Get(scheme)); - } - - public: - const TSchemeInfo& SetScheme(const TSchemeInfo& info) { - SetSchemeImpl(info); + FldSetNoDirty(FieldScheme, info.Str); + return info; + } + const TSchemeInfo& SetSchemeImpl(TScheme::EKind scheme) { + return SetSchemeImpl(TSchemeInfo::Get(scheme)); + } + + public: + const TSchemeInfo& SetScheme(const TSchemeInfo& info) { + SetSchemeImpl(info); if (!info.Str.empty()) - FldMarkClean(FieldScheme); - return info; - } - const TSchemeInfo& SetScheme(TScheme::EKind scheme) { - return SetScheme(TSchemeInfo::Get(scheme)); - } - }; - - class TUriUpdate { - TUri& Uri_; - - public: - TUriUpdate(TUri& uri) - : Uri_(uri) - { - } - ~TUriUpdate() { - Uri_.Rewrite(); - } - - public: - bool Set(TField::EField field, const TStringBuf& value) { - return Uri_.FldMemSet(field, value); - } - - template <size_t size> - bool Set(TField::EField field, const char (&value)[size]) { - return Uri_.FldMemSet(field, value); - } - - void Clr(TField::EField field) { - Uri_.FldClr(field); - } - }; - - const char* LinkTypeToString(const TUri::TLinkType& t); + FldMarkClean(FieldScheme); + return info; + } + const TSchemeInfo& SetScheme(TScheme::EKind scheme) { + return SetScheme(TSchemeInfo::Get(scheme)); + } + }; + + class TUriUpdate { + TUri& Uri_; + + public: + TUriUpdate(TUri& uri) + : Uri_(uri) + { + } + ~TUriUpdate() { + Uri_.Rewrite(); + } + + public: + bool Set(TField::EField field, const TStringBuf& value) { + return Uri_.FldMemSet(field, value); + } + + template <size_t size> + bool Set(TField::EField field, const char (&value)[size]) { + return Uri_.FldMemSet(field, value); + } + + void Clr(TField::EField field) { + Uri_.FldClr(field); + } + }; + + const char* LinkTypeToString(const TUri::TLinkType& t); } -Y_DECLARE_OUT_SPEC(inline, NUri::TUri, out, url) { +Y_DECLARE_OUT_SPEC(inline, NUri::TUri, out, url) { url.Print(out); } -Y_DECLARE_OUT_SPEC(inline, NUri::TUri::TLinkType, out, t) { +Y_DECLARE_OUT_SPEC(inline, NUri::TUri::TLinkType, out, t) { out << NUri::LinkTypeToString(t); } diff --git a/library/cpp/uri/uri_ut.cpp b/library/cpp/uri/uri_ut.cpp index 2ebd83fc93..36fa566ef9 100644 --- a/library/cpp/uri/uri_ut.cpp +++ b/library/cpp/uri/uri_ut.cpp @@ -2,888 +2,888 @@ #include "other.h" #include "qargs.h" #include <library/cpp/html/entity/htmlentity.h> - + #include <util/system/maxlen.h> namespace NUri { Y_UNIT_TEST_SUITE(URLTest) { - static const char* urls[] = { - "http://a/b/c/d;p?q#r", - "g", "http://a/b/c/g", - "./g", "http://a/b/c/g", - "g/", "http://a/b/c/g/", - "/g", "http://a/g", - "//g", "http://g/", - "?y", "http://a/b/c/d;p?y", - "g?y", "http://a/b/c/g?y", - "#s", "http://a/b/c/d;p?q#s", - "g#s", "http://a/b/c/g#s", - "g?y#s", "http://a/b/c/g?y#s", - ";x", "http://a/b/c/;x", - "g;x", "http://a/b/c/g;x", - "g;x?y#s", "http://a/b/c/g;x?y#s", - ".", "http://a/b/c/", - "./", "http://a/b/c/", - "./.", "http://a/b/c/", - "././", "http://a/b/c/", - "././.", "http://a/b/c/", - "..", "http://a/b/", - "../", "http://a/b/", - "../.", "http://a/b/", - "../g", "http://a/b/g", - "../..", "http://a/", - "../../", "http://a/", - "../../.", "http://a/", - "../../g", "http://a/g", - "../../../g", "http://a/g", - "../../../../g", "http://a/g", - "/./g", "http://a/g", - "g.", "http://a/b/c/g.", - ".g", "http://a/b/c/.g", - "g..", "http://a/b/c/g..", - "..g", "http://a/b/c/..g", - "./../g", "http://a/b/g", - "./g/.", "http://a/b/c/g/", - "g/./h", "http://a/b/c/g/h", - "g/../h", "http://a/b/c/h", - "g;x=1/./y", "http://a/b/c/g;x=1/y", - "g;x=1/../y", "http://a/b/c/y", - "g?y/./x", "http://a/b/c/g?y/./x", - "g?y/../x", "http://a/b/c/g?y/../x", - "g#s/./x", "http://a/b/c/g#s/./x", - "g#s/../x", "http://a/b/c/g#s/../x", - "?", "http://a/b/c/d;p?", - "/?", "http://a/?", - "x?", "http://a/b/c/x?", - "x%20y", "http://a/b/c/x%20y", - "%20y", "http://a/b/c/%20y", - // "%2zy", "http://a/b/c/%2zy", - nullptr}; + static const char* urls[] = { + "http://a/b/c/d;p?q#r", + "g", "http://a/b/c/g", + "./g", "http://a/b/c/g", + "g/", "http://a/b/c/g/", + "/g", "http://a/g", + "//g", "http://g/", + "?y", "http://a/b/c/d;p?y", + "g?y", "http://a/b/c/g?y", + "#s", "http://a/b/c/d;p?q#s", + "g#s", "http://a/b/c/g#s", + "g?y#s", "http://a/b/c/g?y#s", + ";x", "http://a/b/c/;x", + "g;x", "http://a/b/c/g;x", + "g;x?y#s", "http://a/b/c/g;x?y#s", + ".", "http://a/b/c/", + "./", "http://a/b/c/", + "./.", "http://a/b/c/", + "././", "http://a/b/c/", + "././.", "http://a/b/c/", + "..", "http://a/b/", + "../", "http://a/b/", + "../.", "http://a/b/", + "../g", "http://a/b/g", + "../..", "http://a/", + "../../", "http://a/", + "../../.", "http://a/", + "../../g", "http://a/g", + "../../../g", "http://a/g", + "../../../../g", "http://a/g", + "/./g", "http://a/g", + "g.", "http://a/b/c/g.", + ".g", "http://a/b/c/.g", + "g..", "http://a/b/c/g..", + "..g", "http://a/b/c/..g", + "./../g", "http://a/b/g", + "./g/.", "http://a/b/c/g/", + "g/./h", "http://a/b/c/g/h", + "g/../h", "http://a/b/c/h", + "g;x=1/./y", "http://a/b/c/g;x=1/y", + "g;x=1/../y", "http://a/b/c/y", + "g?y/./x", "http://a/b/c/g?y/./x", + "g?y/../x", "http://a/b/c/g?y/../x", + "g#s/./x", "http://a/b/c/g#s/./x", + "g#s/../x", "http://a/b/c/g#s/../x", + "?", "http://a/b/c/d;p?", + "/?", "http://a/?", + "x?", "http://a/b/c/x?", + "x%20y", "http://a/b/c/x%20y", + "%20y", "http://a/b/c/%20y", + // "%2zy", "http://a/b/c/%2zy", + nullptr}; Y_UNIT_TEST(test_httpURL) { - TUri rel, base, abs; - TState::EParsed er = base.Parse(urls[0]); - UNIT_ASSERT_VALUES_EQUAL(er, TState::ParsedOK); - UNIT_ASSERT(base.IsValidAbs()); - UNIT_ASSERT_VALUES_EQUAL(base.PrintS(), urls[0]); - - TString errbuf; - TStringOutput out(errbuf); - const long mflag = TFeature::FeaturesAll; - for (int i = 1; urls[i]; i += 2) { - er = rel.Parse(urls[i]); - UNIT_ASSERT_VALUES_EQUAL_C(er, TState::ParsedOK, urls[i]); - rel.Merge(base); - UNIT_ASSERT_VALUES_EQUAL_C(rel.PrintS(), urls[i + 1], urls[i]); - - // try the same thing differently - er = rel.Parse(urls[i], mflag, urls[0]); - UNIT_ASSERT_VALUES_EQUAL_C(er, TState::ParsedOK, urls[i]); - UNIT_ASSERT_VALUES_EQUAL_C(rel.PrintS(), urls[i + 1], urls[i]); - - // lastly... - er = abs.Parse(urls[i + 1], mflag); - UNIT_ASSERT_VALUES_EQUAL(er, TState::ParsedOK); - errbuf.clear(); - out << '[' << rel.PrintS() - << "] != [" << abs.PrintS() << ']'; - UNIT_ASSERT_EQUAL_C(rel, abs, errbuf); - } - } + TUri rel, base, abs; + TState::EParsed er = base.Parse(urls[0]); + UNIT_ASSERT_VALUES_EQUAL(er, TState::ParsedOK); + UNIT_ASSERT(base.IsValidAbs()); + UNIT_ASSERT_VALUES_EQUAL(base.PrintS(), urls[0]); + + TString errbuf; + TStringOutput out(errbuf); + const long mflag = TFeature::FeaturesAll; + for (int i = 1; urls[i]; i += 2) { + er = rel.Parse(urls[i]); + UNIT_ASSERT_VALUES_EQUAL_C(er, TState::ParsedOK, urls[i]); + rel.Merge(base); + UNIT_ASSERT_VALUES_EQUAL_C(rel.PrintS(), urls[i + 1], urls[i]); + + // try the same thing differently + er = rel.Parse(urls[i], mflag, urls[0]); + UNIT_ASSERT_VALUES_EQUAL_C(er, TState::ParsedOK, urls[i]); + UNIT_ASSERT_VALUES_EQUAL_C(rel.PrintS(), urls[i + 1], urls[i]); + + // lastly... + er = abs.Parse(urls[i + 1], mflag); + UNIT_ASSERT_VALUES_EQUAL(er, TState::ParsedOK); + errbuf.clear(); + out << '[' << rel.PrintS() + << "] != [" << abs.PrintS() << ']'; + UNIT_ASSERT_EQUAL_C(rel, abs, errbuf); + } + } Y_UNIT_TEST(test_Schemes) { - TUri url; - UNIT_ASSERT_VALUES_EQUAL(url.Parse("www.ya.ru/index.html"), TState::ParsedOK); - UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeEmpty); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("http://www.ya.ru"), TState::ParsedOK); - UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeHTTP); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("https://www.ya.ru"), TState::ParsedBadScheme); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("https://www.ya.ru", TFeature::FeaturesDefault | TFeature::FeatureSchemeKnown), TState::ParsedOK); - UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeHTTPS); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpwhatever://www.ya.ru"), TState::ParsedBadScheme); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpwhatever://www.ya.ru", TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible), TState::ParsedOK); - UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeUnknown); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpswhatever://www.ya.ru"), TState::ParsedBadScheme); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpswhatever://www.ya.ru", TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible), TState::ParsedOK); - UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeUnknown); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("ftp://www.ya.ru"), TState::ParsedBadScheme); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("ftp://www.ya.ru", TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible), TState::ParsedOK); - UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeFTP); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpsssss://www.ya.ru"), TState::ParsedBadScheme); - UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpsssss://www.ya.ru", TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible), TState::ParsedOK); - UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeUnknown); - } - - struct Link4Norm { - const char* const base; - const char* const link; - const char* const result; - TUri::TLinkType ltype; - }; - - static const Link4Norm link4Norm[] = { - {"http://www.alltest.ru/all.php?a=aberporth", "http://www.alltest.ru/all.php?a=domestic jobs", "", TUri::LinkIsBad}, - {"http://www.alltest.ru/all.php?a=aberporth", "http://www.alltest.ru/all.php?a=domestic%20jobs", "http://www.alltest.ru/all.php?a=domestic%20jobs", TUri::LinkIsLocal}, - {"http://president.rf/%D0%BD%D0%BE%D0%B2%D0%BE%D1%81%D1%82%D0%B8", "http://president.rf/%D0%BD%D0%BE%D0%B2%D0%BE%D1%81%D1%82%D0%B8/1024", "http://president.rf/%D0%BD%D0%BE%D0%B2%D0%BE%D1%81%D1%82%D0%B8/1024", TUri::LinkIsLocal}, - {nullptr, nullptr, nullptr, TUri::LinkIsBad}, - }; + TUri url; + UNIT_ASSERT_VALUES_EQUAL(url.Parse("www.ya.ru/index.html"), TState::ParsedOK); + UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeEmpty); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("http://www.ya.ru"), TState::ParsedOK); + UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeHTTP); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("https://www.ya.ru"), TState::ParsedBadScheme); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("https://www.ya.ru", TFeature::FeaturesDefault | TFeature::FeatureSchemeKnown), TState::ParsedOK); + UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeHTTPS); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpwhatever://www.ya.ru"), TState::ParsedBadScheme); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpwhatever://www.ya.ru", TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible), TState::ParsedOK); + UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeUnknown); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpswhatever://www.ya.ru"), TState::ParsedBadScheme); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpswhatever://www.ya.ru", TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible), TState::ParsedOK); + UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeUnknown); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("ftp://www.ya.ru"), TState::ParsedBadScheme); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("ftp://www.ya.ru", TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible), TState::ParsedOK); + UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeFTP); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpsssss://www.ya.ru"), TState::ParsedBadScheme); + UNIT_ASSERT_VALUES_EQUAL(url.Parse("httpsssss://www.ya.ru", TFeature::FeaturesDefault | TFeature::FeatureSchemeFlexible), TState::ParsedOK); + UNIT_ASSERT_EQUAL(url.GetScheme(), TScheme::SchemeUnknown); + } + + struct Link4Norm { + const char* const base; + const char* const link; + const char* const result; + TUri::TLinkType ltype; + }; + + static const Link4Norm link4Norm[] = { + {"http://www.alltest.ru/all.php?a=aberporth", "http://www.alltest.ru/all.php?a=domestic jobs", "", TUri::LinkIsBad}, + {"http://www.alltest.ru/all.php?a=aberporth", "http://www.alltest.ru/all.php?a=domestic%20jobs", "http://www.alltest.ru/all.php?a=domestic%20jobs", TUri::LinkIsLocal}, + {"http://president.rf/%D0%BD%D0%BE%D0%B2%D0%BE%D1%81%D1%82%D0%B8", "http://president.rf/%D0%BD%D0%BE%D0%B2%D0%BE%D1%81%D1%82%D0%B8/1024", "http://president.rf/%D0%BD%D0%BE%D0%B2%D0%BE%D1%81%D1%82%D0%B8/1024", TUri::LinkIsLocal}, + {nullptr, nullptr, nullptr, TUri::LinkIsBad}, + }; Y_UNIT_TEST(test_httpURLNormalize) { - TUri normalizedLink; - - for (int i = 0; link4Norm[i].link; i++) { - TUri base; - TState::EParsed er = base.Parse(link4Norm[i].base); - UNIT_ASSERT_VALUES_EQUAL_C(er, TState::ParsedOK, link4Norm[i].base); - TUri::TLinkType ltype = normalizedLink.Normalize(base, link4Norm[i].link); - UNIT_ASSERT_VALUES_EQUAL_C(ltype, link4Norm[i].ltype, link4Norm[i].link); - TString s = TUri::LinkIsBad == ltype ? "" : normalizedLink.PrintS(); - UNIT_ASSERT_VALUES_EQUAL_C(s, link4Norm[i].result, link4Norm[i].link); - } - } - - static const char* urlsWithMultipleSlash[] = { - "http://a/http://b", "http://a/http://b", - "http://a/https://b", "http://a/https://b", - "http://a/b://c", "http://a/b:/c", - "http://a/b//c", "http://a/b/c", - nullptr, nullptr}; + TUri normalizedLink; + + for (int i = 0; link4Norm[i].link; i++) { + TUri base; + TState::EParsed er = base.Parse(link4Norm[i].base); + UNIT_ASSERT_VALUES_EQUAL_C(er, TState::ParsedOK, link4Norm[i].base); + TUri::TLinkType ltype = normalizedLink.Normalize(base, link4Norm[i].link); + UNIT_ASSERT_VALUES_EQUAL_C(ltype, link4Norm[i].ltype, link4Norm[i].link); + TString s = TUri::LinkIsBad == ltype ? "" : normalizedLink.PrintS(); + UNIT_ASSERT_VALUES_EQUAL_C(s, link4Norm[i].result, link4Norm[i].link); + } + } + + static const char* urlsWithMultipleSlash[] = { + "http://a/http://b", "http://a/http://b", + "http://a/https://b", "http://a/https://b", + "http://a/b://c", "http://a/b:/c", + "http://a/b//c", "http://a/b/c", + nullptr, nullptr}; Y_UNIT_TEST(test_httpURLPathOperation) { - char copyUrl[URL_MAXLEN]; - for (int i = 0; urlsWithMultipleSlash[i]; i += 2) { - const TStringBuf url(urlsWithMultipleSlash[i]); - const TStringBuf normurl(urlsWithMultipleSlash[i + 1]); - memcpy(copyUrl, url.data(), url.length()); - char* p = copyUrl; - char* e = copyUrl + url.length(); - TUri::PathOperation(p, e, 1); - UNIT_ASSERT_VALUES_EQUAL(TStringBuf(p, e), normurl); - TUri uri; - UNIT_ASSERT_VALUES_EQUAL(TState::ParsedOK, uri.Parse(url)); - UNIT_ASSERT_VALUES_EQUAL_C(uri.PrintS(), normurl, url); - } - } - - static const char* hostsForCheckHost[] = { - "simplehost.ru", - "third_level.host.ru", - "_ok.somewhere.ru", - "a.b", - "second_level.ru", - "_bad.ru", - "_", - "yandex.ru:443", - nullptr}; - - static TState::EParsed answersForCheckHost[] = { - TState::ParsedOK, - TState::ParsedOK, - TState::ParsedOK, - TState::ParsedOK, - TState::ParsedBadHost, - TState::ParsedBadHost, - TState::ParsedBadHost, - TState::ParsedBadHost, - }; + char copyUrl[URL_MAXLEN]; + for (int i = 0; urlsWithMultipleSlash[i]; i += 2) { + const TStringBuf url(urlsWithMultipleSlash[i]); + const TStringBuf normurl(urlsWithMultipleSlash[i + 1]); + memcpy(copyUrl, url.data(), url.length()); + char* p = copyUrl; + char* e = copyUrl + url.length(); + TUri::PathOperation(p, e, 1); + UNIT_ASSERT_VALUES_EQUAL(TStringBuf(p, e), normurl); + TUri uri; + UNIT_ASSERT_VALUES_EQUAL(TState::ParsedOK, uri.Parse(url)); + UNIT_ASSERT_VALUES_EQUAL_C(uri.PrintS(), normurl, url); + } + } + + static const char* hostsForCheckHost[] = { + "simplehost.ru", + "third_level.host.ru", + "_ok.somewhere.ru", + "a.b", + "second_level.ru", + "_bad.ru", + "_", + "yandex.ru:443", + nullptr}; + + static TState::EParsed answersForCheckHost[] = { + TState::ParsedOK, + TState::ParsedOK, + TState::ParsedOK, + TState::ParsedOK, + TState::ParsedBadHost, + TState::ParsedBadHost, + TState::ParsedBadHost, + TState::ParsedBadHost, + }; Y_UNIT_TEST(test_httpURLCheckHost) { - for (size_t index = 0; hostsForCheckHost[index]; ++index) { - TState::EParsed state = TUri::CheckHost(hostsForCheckHost[index]); - UNIT_ASSERT_VALUES_EQUAL(state, answersForCheckHost[index]); - } - } + for (size_t index = 0; hostsForCheckHost[index]; ++index) { + TState::EParsed state = TUri::CheckHost(hostsForCheckHost[index]); + UNIT_ASSERT_VALUES_EQUAL(state, answersForCheckHost[index]); + } + } Y_UNIT_TEST(test_httpURLSet) { - // set port - { - TUri parsedUrl; - parsedUrl.Parse("http://www.host.com/script.cgi?param1=value1¶m2=value2"); - parsedUrl.FldMemSet(TField::FieldPort, "8080"); - UNIT_ASSERT_VALUES_EQUAL(parsedUrl.GetPort(), 8080); - UNIT_ASSERT_VALUES_EQUAL(parsedUrl.PrintS(), "http://www.host.com:8080/script.cgi?param1=value1¶m2=value2"); - } - - // clear port - { - TUri parsedUrl; - parsedUrl.Parse("http://www.host.com:8080/script.cgi?param1=value1¶m2=value2"); - parsedUrl.FldMemSet(TField::FieldPort, nullptr); - UNIT_ASSERT_VALUES_EQUAL(parsedUrl.GetPort(), 80); - UNIT_ASSERT_VALUES_EQUAL(parsedUrl.PrintS(), "http://www.host.com/script.cgi?param1=value1¶m2=value2"); - } - - // change scheme with default port - { - TUri parsedUrl; - parsedUrl.Parse("http://www.host.com/script.cgi?param1=value1¶m2=value2"); - parsedUrl.FldMemSet(TField::FieldScheme, "https"); - UNIT_ASSERT_VALUES_EQUAL(parsedUrl.GetPort(), 443); - UNIT_ASSERT_VALUES_EQUAL(parsedUrl.PrintS(), "https://www.host.com/script.cgi?param1=value1¶m2=value2"); - } - - // change scheme with non-default port - { - TUri parsedUrl; - parsedUrl.Parse("http://www.host.com:8080/script.cgi?param1=value1¶m2=value2"); - parsedUrl.FldMemSet(TField::FieldScheme, "https"); - UNIT_ASSERT_VALUES_EQUAL(parsedUrl.GetPort(), 8080); - UNIT_ASSERT_VALUES_EQUAL(parsedUrl.PrintS(), "https://www.host.com:8080/script.cgi?param1=value1¶m2=value2"); - } - } + // set port + { + TUri parsedUrl; + parsedUrl.Parse("http://www.host.com/script.cgi?param1=value1¶m2=value2"); + parsedUrl.FldMemSet(TField::FieldPort, "8080"); + UNIT_ASSERT_VALUES_EQUAL(parsedUrl.GetPort(), 8080); + UNIT_ASSERT_VALUES_EQUAL(parsedUrl.PrintS(), "http://www.host.com:8080/script.cgi?param1=value1¶m2=value2"); + } + + // clear port + { + TUri parsedUrl; + parsedUrl.Parse("http://www.host.com:8080/script.cgi?param1=value1¶m2=value2"); + parsedUrl.FldMemSet(TField::FieldPort, nullptr); + UNIT_ASSERT_VALUES_EQUAL(parsedUrl.GetPort(), 80); + UNIT_ASSERT_VALUES_EQUAL(parsedUrl.PrintS(), "http://www.host.com/script.cgi?param1=value1¶m2=value2"); + } + + // change scheme with default port + { + TUri parsedUrl; + parsedUrl.Parse("http://www.host.com/script.cgi?param1=value1¶m2=value2"); + parsedUrl.FldMemSet(TField::FieldScheme, "https"); + UNIT_ASSERT_VALUES_EQUAL(parsedUrl.GetPort(), 443); + UNIT_ASSERT_VALUES_EQUAL(parsedUrl.PrintS(), "https://www.host.com/script.cgi?param1=value1¶m2=value2"); + } + + // change scheme with non-default port + { + TUri parsedUrl; + parsedUrl.Parse("http://www.host.com:8080/script.cgi?param1=value1¶m2=value2"); + parsedUrl.FldMemSet(TField::FieldScheme, "https"); + UNIT_ASSERT_VALUES_EQUAL(parsedUrl.GetPort(), 8080); + UNIT_ASSERT_VALUES_EQUAL(parsedUrl.PrintS(), "https://www.host.com:8080/script.cgi?param1=value1¶m2=value2"); + } + } Y_UNIT_TEST(test_httpURLAuth) { - { - TUri parsedUrl; - TState::EParsed st = parsedUrl.Parse("http://@www.host.com/path", TFeature::FeaturesRobot); - UNIT_ASSERT_VALUES_EQUAL(st, TState::ParsedBadAuth); - } - - { - TUri parsedUrl; - TState::EParsed st = parsedUrl.Parse("http://loginwithnopass@www.host.com/path", TFeature::FeatureAuthSupported); - UNIT_ASSERT_VALUES_EQUAL(st, TState::ParsedOK); - UNIT_ASSERT_EQUAL(parsedUrl.GetField(TField::FieldHost), "www.host.com"); - UNIT_ASSERT_EQUAL(parsedUrl.GetField(TField::FieldUser), "loginwithnopass"); - UNIT_ASSERT_EQUAL(parsedUrl.GetField(TField::FieldPass), ""); - } - - { - TUri parsedUrl; - TState::EParsed st = parsedUrl.Parse("http://login:pass@www.host.com/path", TFeature::FeatureAuthSupported); - UNIT_ASSERT_VALUES_EQUAL(st, TState::ParsedOK); - UNIT_ASSERT_EQUAL(parsedUrl.GetField(TField::FieldHost), "www.host.com"); - UNIT_ASSERT_EQUAL(parsedUrl.GetField(TField::FieldUser), "login"); - UNIT_ASSERT_EQUAL(parsedUrl.GetField(TField::FieldPass), "pass"); - } - } + { + TUri parsedUrl; + TState::EParsed st = parsedUrl.Parse("http://@www.host.com/path", TFeature::FeaturesRobot); + UNIT_ASSERT_VALUES_EQUAL(st, TState::ParsedBadAuth); + } + + { + TUri parsedUrl; + TState::EParsed st = parsedUrl.Parse("http://loginwithnopass@www.host.com/path", TFeature::FeatureAuthSupported); + UNIT_ASSERT_VALUES_EQUAL(st, TState::ParsedOK); + UNIT_ASSERT_EQUAL(parsedUrl.GetField(TField::FieldHost), "www.host.com"); + UNIT_ASSERT_EQUAL(parsedUrl.GetField(TField::FieldUser), "loginwithnopass"); + UNIT_ASSERT_EQUAL(parsedUrl.GetField(TField::FieldPass), ""); + } + + { + TUri parsedUrl; + TState::EParsed st = parsedUrl.Parse("http://login:pass@www.host.com/path", TFeature::FeatureAuthSupported); + UNIT_ASSERT_VALUES_EQUAL(st, TState::ParsedOK); + UNIT_ASSERT_EQUAL(parsedUrl.GetField(TField::FieldHost), "www.host.com"); + UNIT_ASSERT_EQUAL(parsedUrl.GetField(TField::FieldUser), "login"); + UNIT_ASSERT_EQUAL(parsedUrl.GetField(TField::FieldPass), "pass"); + } + } Y_UNIT_TEST(test01) { - TTest test = { - "user:pass@host:8080", TFeature::FeaturesAll, TState::ParsedRootless, "user", "", "", "", 0, "", "", ""}; - TUri url; - URL_TEST(url, test); - } + TTest test = { + "user:pass@host:8080", TFeature::FeaturesAll, TState::ParsedRootless, "user", "", "", "", 0, "", "", ""}; + TUri url; + URL_TEST(url, test); + } Y_UNIT_TEST(test02) { - TTest test = { - "http://host", TFeature::FeaturesAll, TState::ParsedOK, "http", "", "", "host", 80, "/", "", ""}; - TUri url; - URL_TEST(url, test); - } + TTest test = { + "http://host", TFeature::FeaturesAll, TState::ParsedOK, "http", "", "", "host", 80, "/", "", ""}; + TUri url; + URL_TEST(url, test); + } Y_UNIT_TEST(test03) { - TTest test = { - "https://host", TFeature::FeatureSchemeFlexible | TFeature::FeatureAllowHostIDN, TState::ParsedOK, "https", "", "", "host", 443, "/", "", ""}; - TUri url; - URL_TEST(url, test); - } + TTest test = { + "https://host", TFeature::FeatureSchemeFlexible | TFeature::FeatureAllowHostIDN, TState::ParsedOK, "https", "", "", "host", 443, "/", "", ""}; + TUri url; + URL_TEST(url, test); + } Y_UNIT_TEST(test04) { - TTest test = { - "user:pass@host:8080", TFeature::FeaturesAll | TFeature::FeatureNoRelPath | TFeature::FeatureAllowRootless, TState::ParsedOK, "user", "", "", "", 0, "pass@host:8080", "", ""}; - TUri url; - URL_TEST(url, test); - TUri url2(url); - CMP_URL(url2, test); - URL_EQ(url, url2); - } + TTest test = { + "user:pass@host:8080", TFeature::FeaturesAll | TFeature::FeatureNoRelPath | TFeature::FeatureAllowRootless, TState::ParsedOK, "user", "", "", "", 0, "pass@host:8080", "", ""}; + TUri url; + URL_TEST(url, test); + TUri url2(url); + CMP_URL(url2, test); + URL_EQ(url, url2); + } Y_UNIT_TEST(test05) { - TTest test = { - "host:8080", TFeature::FeaturesAll | TFeature::FeatureNoRelPath | TFeature::FeatureAllowRootless, TState::ParsedOK, "host", "", "", "", 0, "8080", "", ""}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "host:8080"); - } + TTest test = { + "host:8080", TFeature::FeaturesAll | TFeature::FeatureNoRelPath | TFeature::FeatureAllowRootless, TState::ParsedOK, "host", "", "", "", 0, "8080", "", ""}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "host:8080"); + } Y_UNIT_TEST(test06) { - TTest test = { - "http://user:pass@host?q", TFeature::FeaturesAll, TState::ParsedOK, "http", "user", "pass", "host", 80, "/", "q", ""}; - TUri url; - URL_TEST(url, test); - url.FldMemSet(TField::FieldScheme, "https"); - UNIT_ASSERT(!url.FldIsDirty()); - UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::FieldScheme), "https"); - UNIT_ASSERT_VALUES_EQUAL(url.GetPort(), 443); - - // test copying - TUri url2(url); - // make sure strings are equal... - UNIT_ASSERT_VALUES_EQUAL( - url.GetField(TField::FieldUser), - url2.GetField(TField::FieldUser)); - // ... and memory locations are the same - UNIT_ASSERT_EQUAL( - url.GetField(TField::FieldUser), - url2.GetField(TField::FieldUser)); - // and urls compare the same - URL_EQ(url, url2); - - // cause a dirty field - url.FldMemSet(TField::FieldUser, "use"); // it is now shorter - UNIT_ASSERT(!url.FldIsDirty()); + TTest test = { + "http://user:pass@host?q", TFeature::FeaturesAll, TState::ParsedOK, "http", "user", "pass", "host", 80, "/", "q", ""}; + TUri url; + URL_TEST(url, test); + url.FldMemSet(TField::FieldScheme, "https"); + UNIT_ASSERT(!url.FldIsDirty()); + UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::FieldScheme), "https"); + UNIT_ASSERT_VALUES_EQUAL(url.GetPort(), 443); + + // test copying + TUri url2(url); + // make sure strings are equal... + UNIT_ASSERT_VALUES_EQUAL( + url.GetField(TField::FieldUser), + url2.GetField(TField::FieldUser)); + // ... and memory locations are the same + UNIT_ASSERT_EQUAL( + url.GetField(TField::FieldUser), + url2.GetField(TField::FieldUser)); + // and urls compare the same + URL_EQ(url, url2); + + // cause a dirty field + url.FldMemSet(TField::FieldUser, "use"); // it is now shorter + UNIT_ASSERT(!url.FldIsDirty()); url.FldMemSet(TField::FieldUser, TStringBuf("user")); - UNIT_ASSERT(url.FldIsDirty()); - - // copy again - url2 = url; - UNIT_ASSERT(url.FldIsDirty()); - UNIT_ASSERT(!url2.FldIsDirty()); - URL_EQ(url, url2); - // make sure strings are equal... - UNIT_ASSERT_VALUES_EQUAL( - url.GetField(TField::FieldUser), - url2.GetField(TField::FieldUser)); - // ... but memory locations are different - UNIT_ASSERT_UNEQUAL( - url.GetField(TField::FieldUser).data(), - url2.GetField(TField::FieldUser).data()); - URL_EQ(url, url2); - - // make query empty - url.FldMemSet(TField::FieldQuery, ""); - url2 = url; - URL_EQ(url, url2); - // set query to null value (should clear it) - url2.FldMemSet(TField::FieldQuery, TStringBuf()); - // make sure they are no longer equal - URL_NEQ(url, url2); - // reset query - url.FldClr(TField::FieldQuery); - // equal again - URL_EQ(url, url2); - // reset port and set the other to default - url.FldClr(TField::FieldPort); - url2.FldMemSet(TField::FieldPort, "443"); - URL_EQ(url, url2); - } + UNIT_ASSERT(url.FldIsDirty()); + + // copy again + url2 = url; + UNIT_ASSERT(url.FldIsDirty()); + UNIT_ASSERT(!url2.FldIsDirty()); + URL_EQ(url, url2); + // make sure strings are equal... + UNIT_ASSERT_VALUES_EQUAL( + url.GetField(TField::FieldUser), + url2.GetField(TField::FieldUser)); + // ... but memory locations are different + UNIT_ASSERT_UNEQUAL( + url.GetField(TField::FieldUser).data(), + url2.GetField(TField::FieldUser).data()); + URL_EQ(url, url2); + + // make query empty + url.FldMemSet(TField::FieldQuery, ""); + url2 = url; + URL_EQ(url, url2); + // set query to null value (should clear it) + url2.FldMemSet(TField::FieldQuery, TStringBuf()); + // make sure they are no longer equal + URL_NEQ(url, url2); + // reset query + url.FldClr(TField::FieldQuery); + // equal again + URL_EQ(url, url2); + // reset port and set the other to default + url.FldClr(TField::FieldPort); + url2.FldMemSet(TField::FieldPort, "443"); + URL_EQ(url, url2); + } Y_UNIT_TEST(test07) { - { - TTest test = { - "http://host/path//", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "http", "", "", "host", 80, "/path/", "", ""}; - TUri url; - URL_TEST(url, test); - url.FldMemSet(TField::FieldScheme, "HTTPs"); - UNIT_ASSERT_EQUAL(TScheme::SchemeHTTPS, url.GetScheme()); - UNIT_ASSERT_EQUAL("https", url.GetField(TField::FieldScheme)); - url.FldMemSet(TField::FieldScheme, "HtTP"); - UNIT_ASSERT_EQUAL(TScheme::SchemeHTTP, url.GetScheme()); - UNIT_ASSERT_EQUAL("http", url.GetField(TField::FieldScheme)); - } - - { - const TString scheme = "http"; - const TString host = "host.com"; - const TString urlstr = scheme + "://" + host; - TTest test = { - urlstr, TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, scheme, "", "", host, 80, "/", "", ""}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), urlstr + "/"); - } - } + { + TTest test = { + "http://host/path//", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "http", "", "", "host", 80, "/path/", "", ""}; + TUri url; + URL_TEST(url, test); + url.FldMemSet(TField::FieldScheme, "HTTPs"); + UNIT_ASSERT_EQUAL(TScheme::SchemeHTTPS, url.GetScheme()); + UNIT_ASSERT_EQUAL("https", url.GetField(TField::FieldScheme)); + url.FldMemSet(TField::FieldScheme, "HtTP"); + UNIT_ASSERT_EQUAL(TScheme::SchemeHTTP, url.GetScheme()); + UNIT_ASSERT_EQUAL("http", url.GetField(TField::FieldScheme)); + } + + { + const TString scheme = "http"; + const TString host = "host.com"; + const TString urlstr = scheme + "://" + host; + TTest test = { + urlstr, TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, scheme, "", "", host, 80, "/", "", ""}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), urlstr + "/"); + } + } Y_UNIT_TEST(test08) { - { - TTest test = { - "mailto://user@host.com", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "mailto", "user", "", "host.com", 0, "", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "host:/path/.path/.", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "host", "", "", "", 0, "/path/.path/", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "host:1/path/.path/.", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "", "", "", "host", 1, "/path/.path/", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "host:1/path/.path/.", TFeature::FeaturesAll | TFeature::FeatureAllowRootless, TState::ParsedOK, "host", "", "", "", 0, "1/path/.path/.", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "/[foo]:bar", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "", "", "", "", 0, "/[foo]:bar", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - ".", TFeature::FeaturesAll, TState::ParsedOK, "", "", "", "", 0, "", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - ".", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "", "", "", "", 0, "", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "././.", TFeature::FeaturesAll, TState::ParsedOK, "", "", "", "", 0, "", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "././.", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "", "", "", "", 0, "", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "./path", TFeature::FeaturesAll, TState::ParsedOK, "", "", "", "", 0, "path", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "./path", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "", "", "", "", 0, "path", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "../path", TFeature::FeaturesAll, TState::ParsedOK, "", "", "", "", 0, "../path", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "../path", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "", "", "", "", 0, "../path", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "/../path", TFeature::FeaturesAll, TState::ParsedOK, "", "", "", "", 0, "/path", "", ""}; - TUri url; - URL_TEST(url, test); - } - } + { + TTest test = { + "mailto://user@host.com", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "mailto", "user", "", "host.com", 0, "", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "host:/path/.path/.", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "host", "", "", "", 0, "/path/.path/", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "host:1/path/.path/.", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "", "", "", "host", 1, "/path/.path/", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "host:1/path/.path/.", TFeature::FeaturesAll | TFeature::FeatureAllowRootless, TState::ParsedOK, "host", "", "", "", 0, "1/path/.path/.", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "/[foo]:bar", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "", "", "", "", 0, "/[foo]:bar", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + ".", TFeature::FeaturesAll, TState::ParsedOK, "", "", "", "", 0, "", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + ".", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "", "", "", "", 0, "", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "././.", TFeature::FeaturesAll, TState::ParsedOK, "", "", "", "", 0, "", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "././.", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "", "", "", "", 0, "", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "./path", TFeature::FeaturesAll, TState::ParsedOK, "", "", "", "", 0, "path", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "./path", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "", "", "", "", 0, "path", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "../path", TFeature::FeaturesAll, TState::ParsedOK, "", "", "", "", 0, "../path", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "../path", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "", "", "", "", 0, "../path", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "/../path", TFeature::FeaturesAll, TState::ParsedOK, "", "", "", "", 0, "/path", "", ""}; + TUri url; + URL_TEST(url, test); + } + } Y_UNIT_TEST(test09) { - { - TTest test = { - "mailto:user@host.com", TFeature::FeaturesAll | TFeature::FeatureAllowRootless, TState::ParsedOK, "mailto", "", "", "", 0, "user@host.com", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "scheme:", TFeature::FeaturesAll | TFeature::FeatureNoRelPath | TFeature::FeatureAllowRootless, TState::ParsedOK, "scheme", "", "", "", 0, "", "", ""}; - TUri url; - URL_TEST(url, test); - } - { - TTest test = { - "scheme:", TFeature::FeaturesAll | TFeature::FeatureAllowRootless, TState::ParsedOK, "scheme", "", "", "", 0, "", "", ""}; - TUri url; - URL_TEST(url, test); - } - } + { + TTest test = { + "mailto:user@host.com", TFeature::FeaturesAll | TFeature::FeatureAllowRootless, TState::ParsedOK, "mailto", "", "", "", 0, "user@host.com", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "scheme:", TFeature::FeaturesAll | TFeature::FeatureNoRelPath | TFeature::FeatureAllowRootless, TState::ParsedOK, "scheme", "", "", "", 0, "", "", ""}; + TUri url; + URL_TEST(url, test); + } + { + TTest test = { + "scheme:", TFeature::FeaturesAll | TFeature::FeatureAllowRootless, TState::ParsedOK, "scheme", "", "", "", 0, "", "", ""}; + TUri url; + URL_TEST(url, test); + } + } Y_UNIT_TEST(test10) { - // test some escaping madness, note the ehost vs host - { - TString host = "президент.рф"; - TString ehost = "%D0%BF%D1%80%D0%B5%D0%B7%D0%B8%D0%B4%D0%B5%D0%BD%D1%82.%D1%80%D1%84"; - const TString urlstr = TString::Join("http://", host, "/"); - TTest test = { - urlstr, TFeature::FeatureEncodeExtendedASCII | TFeature::FeaturesDefault | TFeature::FeatureCheckHost, TState::ParsedBadHost, "http", "", "", ehost, 80, "/", "", ""}; - TUri url; - URL_TEST(url, test); - } - - { - TString host = "%D0%BF%D1%80%D0%B5%D0%B7%D0%B8%D0%B4%D0%B5%D0%BD%D1%82.%D1%80%D1%84"; - const TString urlstr = TString::Join("http://", host, "/"); - TTest test = { - urlstr, TFeature::FeatureEncodeExtendedASCII | TFeature::FeaturesDefault | TFeature::FeatureCheckHost, TState::ParsedBadHost, "http", "", "", host, 80, "/", "", ""}; - TUri url; - URL_TEST(url, test); - } - - { - TString host = "Фilip.ru"; - TString ehost = "%D0%A4ilip.ru"; - const TString urlstr = TString::Join("http://", host); - TTest test = { - urlstr, TFeature::FeatureEncodeExtendedASCII | TFeature::FeaturesDefault, TState::ParsedBadHost, "http", "", "", ehost, 80, "/", "", ""}; - TUri url; - URL_TEST(url, test); - } - - { - TString host = "%D0%A4ilip.ru"; - const TString urlstr = TString::Join("http://", host); - TTest test = { - urlstr, TFeature::FeatureEncodeExtendedASCII | TFeature::FeaturesDefault, TState::ParsedBadHost, "http", "", "", host, 80, "/", "", ""}; - TUri url; - URL_TEST(url, test); - } - - { - TString host = "Filip%90.rЯ"; - TString ehost = "Filip%90.r%D0%AF"; - const TString urlstr = TString::Join(host, ":8080"); - TTest test = { - urlstr, TFeature::FeatureEncodeExtendedASCII | TFeature::FeatureDecodeAllowed | TFeature::FeaturesDefault | TFeature::FeatureNoRelPath, TState::ParsedBadHost, "", "", "", ehost, 8080, "", "", ""}; - TUri url; - URL_TEST(url, test); - } - - { - TString host = "Filip%90.r%D0%AF"; - const TString urlstr = TString::Join(host, ":8080"); - TTest test = { - urlstr, TFeature::FeatureEncodeExtendedASCII | TFeature::FeatureDecodeAllowed | TFeature::FeaturesDefault | TFeature::FeatureNoRelPath, TState::ParsedBadHost, "", "", "", host, 8080, "", "", ""}; - TUri url; - URL_TEST(url, test); - } - } + // test some escaping madness, note the ehost vs host + { + TString host = "президент.рф"; + TString ehost = "%D0%BF%D1%80%D0%B5%D0%B7%D0%B8%D0%B4%D0%B5%D0%BD%D1%82.%D1%80%D1%84"; + const TString urlstr = TString::Join("http://", host, "/"); + TTest test = { + urlstr, TFeature::FeatureEncodeExtendedASCII | TFeature::FeaturesDefault | TFeature::FeatureCheckHost, TState::ParsedBadHost, "http", "", "", ehost, 80, "/", "", ""}; + TUri url; + URL_TEST(url, test); + } + + { + TString host = "%D0%BF%D1%80%D0%B5%D0%B7%D0%B8%D0%B4%D0%B5%D0%BD%D1%82.%D1%80%D1%84"; + const TString urlstr = TString::Join("http://", host, "/"); + TTest test = { + urlstr, TFeature::FeatureEncodeExtendedASCII | TFeature::FeaturesDefault | TFeature::FeatureCheckHost, TState::ParsedBadHost, "http", "", "", host, 80, "/", "", ""}; + TUri url; + URL_TEST(url, test); + } + + { + TString host = "Фilip.ru"; + TString ehost = "%D0%A4ilip.ru"; + const TString urlstr = TString::Join("http://", host); + TTest test = { + urlstr, TFeature::FeatureEncodeExtendedASCII | TFeature::FeaturesDefault, TState::ParsedBadHost, "http", "", "", ehost, 80, "/", "", ""}; + TUri url; + URL_TEST(url, test); + } + + { + TString host = "%D0%A4ilip.ru"; + const TString urlstr = TString::Join("http://", host); + TTest test = { + urlstr, TFeature::FeatureEncodeExtendedASCII | TFeature::FeaturesDefault, TState::ParsedBadHost, "http", "", "", host, 80, "/", "", ""}; + TUri url; + URL_TEST(url, test); + } + + { + TString host = "Filip%90.rЯ"; + TString ehost = "Filip%90.r%D0%AF"; + const TString urlstr = TString::Join(host, ":8080"); + TTest test = { + urlstr, TFeature::FeatureEncodeExtendedASCII | TFeature::FeatureDecodeAllowed | TFeature::FeaturesDefault | TFeature::FeatureNoRelPath, TState::ParsedBadHost, "", "", "", ehost, 8080, "", "", ""}; + TUri url; + URL_TEST(url, test); + } + + { + TString host = "Filip%90.r%D0%AF"; + const TString urlstr = TString::Join(host, ":8080"); + TTest test = { + urlstr, TFeature::FeatureEncodeExtendedASCII | TFeature::FeatureDecodeAllowed | TFeature::FeaturesDefault | TFeature::FeatureNoRelPath, TState::ParsedBadHost, "", "", "", host, 8080, "", "", ""}; + TUri url; + URL_TEST(url, test); + } + } Y_UNIT_TEST(test11) { - { - TTest test = { - "HtTp://HoSt/%50aTh/?Query#Frag", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "http", "", "", "host", 80, "/PaTh/", "Query", "Frag"}; - TUri url; - URL_TEST(url, test); - } - - { - TTest test = { - "HtTp://HoSt/%50a%54h/?Query#Frag", TParseFlags(TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TFeature::FeatureToLower), TState::ParsedOK, "http", "", "", "host", 80, "/path/", "query", "frag"}; - TUri url; - URL_TEST(url, test); - } - } + { + TTest test = { + "HtTp://HoSt/%50aTh/?Query#Frag", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedOK, "http", "", "", "host", 80, "/PaTh/", "Query", "Frag"}; + TUri url; + URL_TEST(url, test); + } + + { + TTest test = { + "HtTp://HoSt/%50a%54h/?Query#Frag", TParseFlags(TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TFeature::FeatureToLower), TState::ParsedOK, "http", "", "", "host", 80, "/path/", "query", "frag"}; + TUri url; + URL_TEST(url, test); + } + } Y_UNIT_TEST(test12) { - // test characters which are not always safe - { + // test characters which are not always safe + { #define RAW "/:" #define DEC "%2F:" #define ENC "%2F%3A" - TTest test = { - "http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC, TFeature::FeaturesAll, TState::ParsedOK, "http", RAW, RAW, "host", 80, "/" DEC, RAW, RAW}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" ENC ":" ENC "@host/" DEC "?" RAW "#" RAW); + TTest test = { + "http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC, TFeature::FeaturesAll, TState::ParsedOK, "http", RAW, RAW, "host", 80, "/" DEC, RAW, RAW}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" ENC ":" ENC "@host/" DEC "?" RAW "#" RAW); #undef RAW #undef DEC #undef ENC - } - { + } + { #define RAW "?@" #define DEC "%3F@" #define ENC "%3F%40" - TTest test = { - "http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC, TFeature::FeaturesAll, TState::ParsedOK, "http", RAW, RAW, "host", 80, "/" DEC, RAW, RAW}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" ENC ":" ENC "@host/" DEC "?" RAW "#" RAW); + TTest test = { + "http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC, TFeature::FeaturesAll, TState::ParsedOK, "http", RAW, RAW, "host", 80, "/" DEC, RAW, RAW}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" ENC ":" ENC "@host/" DEC "?" RAW "#" RAW); #undef RAW #undef DEC #undef ENC - } - { + } + { #define RAW "%&;=" #define DEC "%25&;=" #define ENC "%25%26%3B%3D" - TTest test = { - "http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC, TFeature::FeaturesAll, TState::ParsedOK, "http", RAW, RAW, "host", 80, "/" ENC, ENC, ENC}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC); + TTest test = { + "http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC, TFeature::FeaturesAll, TState::ParsedOK, "http", RAW, RAW, "host", 80, "/" ENC, ENC, ENC}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC); #undef RAW #undef DEC #undef ENC - } - { + } + { #define RAW "!$'()*," #define DEC "!$%27()*," #define ENC "%21%24%27%28%29%2A%2C" - TTest test = { - "http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC, TFeature::FeaturesAll, TState::ParsedOK, "http", RAW, RAW, "host", 80, "/" ENC, DEC, DEC}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" ENC ":" ENC "@host/" ENC "?" DEC "#" DEC); + TTest test = { + "http://" ENC ":" ENC "@host/" ENC "?" ENC "#" ENC, TFeature::FeaturesAll, TState::ParsedOK, "http", RAW, RAW, "host", 80, "/" ENC, DEC, DEC}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" ENC ":" ENC "@host/" ENC "?" DEC "#" DEC); #undef RAW #undef DEC #undef ENC - } - { + } + { #define DEC "Череповец。рф" #define ENC "%D0%A7%D0%B5%D1%80%D0%B5%D0%BF%D0%BE%D0%B2%D0%B5%D1%86%E3%80%82%D1%80%D1%84" // punycode corresponds to lowercase #define PNC "xn--b1afab7bff7cb.xn--p1ai" - TTest test = { - "http://" ENC "/" ENC "?" ENC "#" ENC, TParseFlags(TFeature::FeaturesAll | TFeature::FeatureAllowHostIDN, TFeature::FeatureDecodeExtendedASCII), TState::ParsedOK, "http", "", "", DEC, 80, "/" ENC, ENC, ENC}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::FieldHostAscii), PNC); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" DEC "/" ENC "?" ENC "#" ENC); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagHostAscii), "http://" PNC "/" ENC "?" ENC "#" ENC); + TTest test = { + "http://" ENC "/" ENC "?" ENC "#" ENC, TParseFlags(TFeature::FeaturesAll | TFeature::FeatureAllowHostIDN, TFeature::FeatureDecodeExtendedASCII), TState::ParsedOK, "http", "", "", DEC, 80, "/" ENC, ENC, ENC}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::FieldHostAscii), PNC); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" DEC "/" ENC "?" ENC "#" ENC); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(TField::FlagHostAscii), "http://" PNC "/" ENC "?" ENC "#" ENC); #undef PNC #undef DEC #undef ENC - } - { + } + { #define DEC "Череповец。рф" #define ENC "%D0%A7%D0%B5%D1%80%D0%B5%D0%BF%D0%BE%D0%B2%D0%B5%D1%86%E3%80%82%D1%80%D1%84" // punycode corresponds to lowercase #define PNC "xn--b1afab7bff7cb.xn--p1ai" - TTest test = { - "http://" DEC "/" DEC "?" DEC "#" DEC, TParseFlags(TFeature::FeaturesRobot | TFeature::FeatureEncodeExtendedASCII), TState::ParsedOK, "http", "", "", PNC, 80, "/" ENC, ENC, ENC}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" PNC "/" ENC "?" ENC "#" ENC); + TTest test = { + "http://" DEC "/" DEC "?" DEC "#" DEC, TParseFlags(TFeature::FeaturesRobot | TFeature::FeatureEncodeExtendedASCII), TState::ParsedOK, "http", "", "", PNC, 80, "/" ENC, ENC, ENC}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" PNC "/" ENC "?" ENC "#" ENC); #undef PNC #undef DEC #undef ENC - } - { + } + { #define DEC "независимая-экспертиза-оценка-ущерба-авто-дтп.рф" #define PNC "xn--------3veabbbbjgk5abecc3afsad2cg8bvq2alouolqf5brd3a4jzftgqd.xn--p1ai" - TTest test = { - "http://" DEC "/", TParseFlags(TFeature::FeaturesRobot), TState::ParsedOK, "http", "", "", PNC, 80, "/", "", ""}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" PNC "/"); + TTest test = { + "http://" DEC "/", TParseFlags(TFeature::FeaturesRobot), TState::ParsedOK, "http", "", "", PNC, 80, "/", "", ""}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" PNC "/"); #undef PNC #undef DEC - } - } + } + } Y_UNIT_TEST(testFlexibleAuthority) { - TUri uri; - UNIT_ASSERT_EQUAL(uri.Parse("http://hello_world", TFeature::FeatureCheckHost), TState::ParsedBadHost); - UNIT_ASSERT_EQUAL(uri.Parse("http://hello_world", TFeature::FeatureSchemeFlexible), TState::ParsedOK); - UNIT_ASSERT_VALUES_EQUAL(uri.GetHost(), "hello_world"); - - UNIT_ASSERT_EQUAL(uri.Parse("httpzzzzz://)(*&^$!\\][';<>`~,q?./index.html", TFeature::FeatureSchemeFlexible), TState::ParsedOK); - UNIT_ASSERT_VALUES_EQUAL(uri.GetHost(), ")(*&^$!\\][';<>`~,q"); - UNIT_ASSERT_VALUES_EQUAL(uri.GetField(TField::FieldPath), ""); - UNIT_ASSERT_VALUES_EQUAL(uri.GetField(TField::FieldQuery), "./index.html"); - - UNIT_ASSERT_EQUAL(uri.Parse("htttttttp://)(*&^%45$!\\][';<>`~,.q/index.html", TFeature::FeatureSchemeFlexible), TState::ParsedOK); - UNIT_ASSERT_VALUES_EQUAL(uri.GetHost(), ")(*&^e$!\\][';<>`~,.q"); - UNIT_ASSERT_VALUES_EQUAL(uri.GetField(TField::FieldPath), "/index.html"); - UNIT_ASSERT_VALUES_EQUAL(uri.GetField(TField::FieldQuery), ""); - } + TUri uri; + UNIT_ASSERT_EQUAL(uri.Parse("http://hello_world", TFeature::FeatureCheckHost), TState::ParsedBadHost); + UNIT_ASSERT_EQUAL(uri.Parse("http://hello_world", TFeature::FeatureSchemeFlexible), TState::ParsedOK); + UNIT_ASSERT_VALUES_EQUAL(uri.GetHost(), "hello_world"); + + UNIT_ASSERT_EQUAL(uri.Parse("httpzzzzz://)(*&^$!\\][';<>`~,q?./index.html", TFeature::FeatureSchemeFlexible), TState::ParsedOK); + UNIT_ASSERT_VALUES_EQUAL(uri.GetHost(), ")(*&^$!\\][';<>`~,q"); + UNIT_ASSERT_VALUES_EQUAL(uri.GetField(TField::FieldPath), ""); + UNIT_ASSERT_VALUES_EQUAL(uri.GetField(TField::FieldQuery), "./index.html"); + + UNIT_ASSERT_EQUAL(uri.Parse("htttttttp://)(*&^%45$!\\][';<>`~,.q/index.html", TFeature::FeatureSchemeFlexible), TState::ParsedOK); + UNIT_ASSERT_VALUES_EQUAL(uri.GetHost(), ")(*&^e$!\\][';<>`~,.q"); + UNIT_ASSERT_VALUES_EQUAL(uri.GetField(TField::FieldPath), "/index.html"); + UNIT_ASSERT_VALUES_EQUAL(uri.GetField(TField::FieldQuery), ""); + } Y_UNIT_TEST(testSpecialChar) { - // test characters which are not always allowed - { - TTest test = { - "http://host/pa th", TFeature::FeaturesAll | TFeature::FeatureEncodeSpace, TState::ParsedOK, "http", "", "", "host", 80, "/pa%20th", "", ""}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/pa%20th"); - } - { - TTest test = { - "http://host/pa th", TFeature::FeaturesAll, TState::ParsedBadFormat, "http", "", "", "host", 80, "/pa th", "", ""}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/pa th"); - } - { - TTest test = { - "http://host/pa%th%41", TFeature::FeaturesAll | TFeature::FeatureEncodePercent, TState::ParsedOK, "http", "", "", "host", 80, "/pa%25thA", "", ""}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/pa%25thA"); - } - { - TTest test = { - "http://host/invalid_second_char%az%1G", TFeature::FeaturesAll | TFeature::FeatureEncodePercent, TState::ParsedOK, "http", "", "", "host", 80, "/invalid_second_char%25az%251G", "", ""}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/invalid_second_char%25az%251G"); - } - { - TTest test = { - "http://host/border%2", TFeature::FeaturesAll | TFeature::FeatureEncodePercent, TState::ParsedOK, "http", "", "", "host", 80, "/border%252", "", ""}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/border%252"); - } - { - TTest test = { - "http://host/pa%th%41", TFeature::FeaturesAll, TState::ParsedBadFormat, "http", "", "", "host", 80, "/pa%thA", "", ""}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/pa%thA"); - } - } + // test characters which are not always allowed + { + TTest test = { + "http://host/pa th", TFeature::FeaturesAll | TFeature::FeatureEncodeSpace, TState::ParsedOK, "http", "", "", "host", 80, "/pa%20th", "", ""}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/pa%20th"); + } + { + TTest test = { + "http://host/pa th", TFeature::FeaturesAll, TState::ParsedBadFormat, "http", "", "", "host", 80, "/pa th", "", ""}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/pa th"); + } + { + TTest test = { + "http://host/pa%th%41", TFeature::FeaturesAll | TFeature::FeatureEncodePercent, TState::ParsedOK, "http", "", "", "host", 80, "/pa%25thA", "", ""}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/pa%25thA"); + } + { + TTest test = { + "http://host/invalid_second_char%az%1G", TFeature::FeaturesAll | TFeature::FeatureEncodePercent, TState::ParsedOK, "http", "", "", "host", 80, "/invalid_second_char%25az%251G", "", ""}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/invalid_second_char%25az%251G"); + } + { + TTest test = { + "http://host/border%2", TFeature::FeaturesAll | TFeature::FeatureEncodePercent, TState::ParsedOK, "http", "", "", "host", 80, "/border%252", "", ""}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/border%252"); + } + { + TTest test = { + "http://host/pa%th%41", TFeature::FeaturesAll, TState::ParsedBadFormat, "http", "", "", "host", 80, "/pa%thA", "", ""}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host/pa%thA"); + } + } Y_UNIT_TEST(testIPv6) { - { + { #define RAW "[1080:0:0:0:8:800:200C:417A]" #define DEC "[1080:0:0:0:8:800:200c:417a]" - TTest test = { - "http://" RAW "/" RAW "?" RAW "#" RAW, TParseFlags(TFeature::FeaturesAll), TState::ParsedOK, "http", "", "", DEC, 80, "/" RAW, RAW, RAW}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" DEC "/" RAW "?" RAW "#" RAW); + TTest test = { + "http://" RAW "/" RAW "?" RAW "#" RAW, TParseFlags(TFeature::FeaturesAll), TState::ParsedOK, "http", "", "", DEC, 80, "/" RAW, RAW, RAW}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://" DEC "/" RAW "?" RAW "#" RAW); #undef DEC #undef RAW - } - } + } + } Y_UNIT_TEST(testEscapedFragment) { - { - TTest test = { - "http://host.com#!a=b&c=d#e+g%41%25", TParseFlags(TFeature::FeaturesAll | TFeature::FeatureHashBangToEscapedFragment), TState::ParsedOK, "http", "", "", "host.com", 80, "/", "_escaped_fragment_=a=b%26c=d%23e%2BgA%2525", ""}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host.com/?_escaped_fragment_=a=b%26c=d%23e%2BgA%2525"); - } - { - TTest test = { - "http://host.com?_escaped_fragment_=a=b%26c=d%23e%2bg%2525", TParseFlags(TFeature::FeaturesAll | TFeature::FeatureEscapedToHashBangFragment), TState::ParsedOK, "http", "", "", "host.com", 80, "/", "", "!a=b&c=d#e+g%25"}; - TUri url; - URL_TEST(url, test); - UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host.com/#!a=b&c=d#e+g%25"); - } - } + { + TTest test = { + "http://host.com#!a=b&c=d#e+g%41%25", TParseFlags(TFeature::FeaturesAll | TFeature::FeatureHashBangToEscapedFragment), TState::ParsedOK, "http", "", "", "host.com", 80, "/", "_escaped_fragment_=a=b%26c=d%23e%2BgA%2525", ""}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host.com/?_escaped_fragment_=a=b%26c=d%23e%2BgA%2525"); + } + { + TTest test = { + "http://host.com?_escaped_fragment_=a=b%26c=d%23e%2bg%2525", TParseFlags(TFeature::FeaturesAll | TFeature::FeatureEscapedToHashBangFragment), TState::ParsedOK, "http", "", "", "host.com", 80, "/", "", "!a=b&c=d#e+g%25"}; + TUri url; + URL_TEST(url, test); + UNIT_ASSERT_VALUES_EQUAL(url.PrintS(), "http://host.com/#!a=b&c=d#e+g%25"); + } + } Y_UNIT_TEST(testReEncode) { - { - TStringStream out; - TUri::ReEncode(out, "foo bar"); - UNIT_ASSERT_VALUES_EQUAL(out.Str(), "foo%20bar"); - } - } - - static const TStringBuf NonRfcUrls[] = { - "http://deshevle.ru/price/price=&SrchTp=1&clID=24&BL=SrchTp=0|clID=24&frmID=75&SortBy=P&PreSort=&NmDir=0&VndDir=0&PrDir=0&SPP=44", - "http://secure.rollerwarehouse.com/skates/aggressive/skates/c/11[03]/tx/$$$+11[03][a-z]", - "http://secure.rollerwarehouse.com/skates/aggressive/skates/tx/$$$+110[a-z]", - "http://translate.google.com/translate_t?langpair=en|ru", - "http://www.garnier.com.ru/_ru/_ru/our_products/products_trade.aspx?tpcode=OUR_PRODUCTS^PRD_BODYCARE^EXTRA_SKIN^EXTRA_SKIN_BENEFITS", - "http://www.km.ru/magazin/view_print.asp?id={1846295A-223B-41DC-9F51-90D5D6236C49}", - "http://www.manutd.com/default.sps?pagegid={78F24B85-702C-4DC8-A5D4-2F67252C28AA}&itype=12977&pagebuildpageid=2716&bg=1", - "http://www.pokupay.ru/price/price=&SrchTp=1&clID=24&BL=SrchTp=0|clID=24&frmID=75&SPP=35&SortBy=N&PreSort=V&NmDir=0&VndDir=1&PrDir=0", - "http://www.rodnoyspb.ru/rest/plager/page[0].html", - "http://www.trinity.by/?section_id=46,47,48&cat=1&filters[]=2^_^Sony", - "http://translate.yandex.net/api/v1/tr.json/translate?lang=en-ru&text=>", - nullptr}; + { + TStringStream out; + TUri::ReEncode(out, "foo bar"); + UNIT_ASSERT_VALUES_EQUAL(out.Str(), "foo%20bar"); + } + } + + static const TStringBuf NonRfcUrls[] = { + "http://deshevle.ru/price/price=&SrchTp=1&clID=24&BL=SrchTp=0|clID=24&frmID=75&SortBy=P&PreSort=&NmDir=0&VndDir=0&PrDir=0&SPP=44", + "http://secure.rollerwarehouse.com/skates/aggressive/skates/c/11[03]/tx/$$$+11[03][a-z]", + "http://secure.rollerwarehouse.com/skates/aggressive/skates/tx/$$$+110[a-z]", + "http://translate.google.com/translate_t?langpair=en|ru", + "http://www.garnier.com.ru/_ru/_ru/our_products/products_trade.aspx?tpcode=OUR_PRODUCTS^PRD_BODYCARE^EXTRA_SKIN^EXTRA_SKIN_BENEFITS", + "http://www.km.ru/magazin/view_print.asp?id={1846295A-223B-41DC-9F51-90D5D6236C49}", + "http://www.manutd.com/default.sps?pagegid={78F24B85-702C-4DC8-A5D4-2F67252C28AA}&itype=12977&pagebuildpageid=2716&bg=1", + "http://www.pokupay.ru/price/price=&SrchTp=1&clID=24&BL=SrchTp=0|clID=24&frmID=75&SPP=35&SortBy=N&PreSort=V&NmDir=0&VndDir=1&PrDir=0", + "http://www.rodnoyspb.ru/rest/plager/page[0].html", + "http://www.trinity.by/?section_id=46,47,48&cat=1&filters[]=2^_^Sony", + "http://translate.yandex.net/api/v1/tr.json/translate?lang=en-ru&text=>", + nullptr}; Y_UNIT_TEST(test_NonRfcUrls) { - TUri url; - const long flags = TFeature::FeaturesRobot; - for (size_t i = 0;; ++i) { - const TStringBuf& buf = NonRfcUrls[i]; - if (!buf.IsInited()) - break; - UNIT_ASSERT_VALUES_EQUAL(TState::ParsedOK, url.Parse(buf, flags)); - } - } - - static const TStringBuf CheckParseException[] = { - "http://www.'>'.com/?.net/", - nullptr}; + TUri url; + const long flags = TFeature::FeaturesRobot; + for (size_t i = 0;; ++i) { + const TStringBuf& buf = NonRfcUrls[i]; + if (!buf.IsInited()) + break; + UNIT_ASSERT_VALUES_EQUAL(TState::ParsedOK, url.Parse(buf, flags)); + } + } + + static const TStringBuf CheckParseException[] = { + "http://www.'>'.com/?.net/", + nullptr}; Y_UNIT_TEST(test_CheckParseException) { - TUri url; - const long flags = TFeature::FeaturesRobot | TFeature::FeaturesEncode; - for (size_t i = 0;; ++i) { - const TStringBuf& buf = CheckParseException[i]; - if (!buf.IsInited()) - break; - TString what; - try { - // we care only about exceptions, not whether it parses correctly - url.Parse(buf, flags); - continue; - } catch (const std::exception& exc) { - what = exc.what(); - } catch (...) { - what = "exception thrown"; - } - ythrow yexception() << "failed to parse URL [" << buf << "]: " << what; - } + TUri url; + const long flags = TFeature::FeaturesRobot | TFeature::FeaturesEncode; + for (size_t i = 0;; ++i) { + const TStringBuf& buf = CheckParseException[i]; + if (!buf.IsInited()) + break; + TString what; + try { + // we care only about exceptions, not whether it parses correctly + url.Parse(buf, flags); + continue; + } catch (const std::exception& exc) { + what = exc.what(); + } catch (...) { + what = "exception thrown"; + } + ythrow yexception() << "failed to parse URL [" << buf << "]: " << what; + } } Y_UNIT_TEST(test_PrintPort) { - TUri uri; - { - uri.Parse("http://srv.net:9100/print", TFeature::FeaturesRecommended); - TString s = uri.PrintS(TUri::FlagPort); - Cdbg << uri.PrintS() << ',' << uri.PrintS(TUri::FlagPort) << Endl; - UNIT_ASSERT_VALUES_EQUAL(9100, FromString<ui32>(s)); - } - { - uri.Parse("http://srv.net:80/print", TFeature::FeaturesRecommended); - TString s = uri.PrintS(TUri::FlagPort); - Cdbg << uri.PrintS() << ',' << uri.PrintS(TUri::FlagPort) << Endl; - UNIT_ASSERT(s.Empty()); - } - } - + TUri uri; + { + uri.Parse("http://srv.net:9100/print", TFeature::FeaturesRecommended); + TString s = uri.PrintS(TUri::FlagPort); + Cdbg << uri.PrintS() << ',' << uri.PrintS(TUri::FlagPort) << Endl; + UNIT_ASSERT_VALUES_EQUAL(9100, FromString<ui32>(s)); + } + { + uri.Parse("http://srv.net:80/print", TFeature::FeaturesRecommended); + TString s = uri.PrintS(TUri::FlagPort); + Cdbg << uri.PrintS() << ',' << uri.PrintS(TUri::FlagPort) << Endl; + UNIT_ASSERT(s.Empty()); + } + } + Y_UNIT_TEST(test_ParseFailures) { - { - TTest test = { - "http://host:port", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedBadFormat, "", "", "", "", Max<ui16>(), "", "", ""}; - TUri url(-1); - URL_TEST(url, test); - } - { - TTest test = { - "http://javascript:alert(hi)", TFeature::FeaturesRobot, TState::ParsedBadFormat, "", "", "", "", Max<ui16>(), "", "", ""}; - TUri url(-1); - URL_TEST(url, test); - } - { - TTest test = { - "http://host::0", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedBadFormat, "", "", "", "", Max<ui16>(), "", "", ""}; - TUri url(-1); - URL_TEST(url, test); - } - { - TTest test = { - "http://host ", TFeature::FeaturesAll, TState::ParsedBadFormat, "", "", "", "", Max<ui16>(), "", "", ""}; - TUri url(-1); - URL_TEST(url, test); - } - { - TTest test = { - "http:00..03", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedBadFormat, "", "", "", "", Max<ui16>(), "", "", ""}; - TUri url(-1); - URL_TEST(url, test); - } - { - TTest test = { - "host:00..03", TFeature::FeaturesAll, TState::ParsedRootless, "host", "", "", "", 0, "", "", ""}; - TUri url(-1); - URL_TEST(url, test); - } - { - TTest test = { - "http://roduct;isbn,0307371549;at,aid4c00179ab018www.mcnamarasband.wordpress.com/", TFeature::FeaturesAll, TState::ParsedBadHost, "http", "", "", "roduct;isbn,0307371549;at,aid4c00179ab018www.mcnamarasband.wordpress.com", 80, "/", "", ""}; - TUri url(-1); - URL_TEST(url, test); - } - { - TTest test = { - "invalid url", TFeature::FeaturesDefault, TState::ParsedBadFormat, "", "", "", "", 0, "invalid url", "", ""}; - TUri url(-1); - URL_TEST(url, test); - } - } + { + TTest test = { + "http://host:port", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedBadFormat, "", "", "", "", Max<ui16>(), "", "", ""}; + TUri url(-1); + URL_TEST(url, test); + } + { + TTest test = { + "http://javascript:alert(hi)", TFeature::FeaturesRobot, TState::ParsedBadFormat, "", "", "", "", Max<ui16>(), "", "", ""}; + TUri url(-1); + URL_TEST(url, test); + } + { + TTest test = { + "http://host::0", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedBadFormat, "", "", "", "", Max<ui16>(), "", "", ""}; + TUri url(-1); + URL_TEST(url, test); + } + { + TTest test = { + "http://host ", TFeature::FeaturesAll, TState::ParsedBadFormat, "", "", "", "", Max<ui16>(), "", "", ""}; + TUri url(-1); + URL_TEST(url, test); + } + { + TTest test = { + "http:00..03", TFeature::FeaturesAll | TFeature::FeatureNoRelPath, TState::ParsedBadFormat, "", "", "", "", Max<ui16>(), "", "", ""}; + TUri url(-1); + URL_TEST(url, test); + } + { + TTest test = { + "host:00..03", TFeature::FeaturesAll, TState::ParsedRootless, "host", "", "", "", 0, "", "", ""}; + TUri url(-1); + URL_TEST(url, test); + } + { + TTest test = { + "http://roduct;isbn,0307371549;at,aid4c00179ab018www.mcnamarasband.wordpress.com/", TFeature::FeaturesAll, TState::ParsedBadHost, "http", "", "", "roduct;isbn,0307371549;at,aid4c00179ab018www.mcnamarasband.wordpress.com", 80, "/", "", ""}; + TUri url(-1); + URL_TEST(url, test); + } + { + TTest test = { + "invalid url", TFeature::FeaturesDefault, TState::ParsedBadFormat, "", "", "", "", 0, "invalid url", "", ""}; + TUri url(-1); + URL_TEST(url, test); + } + } Y_UNIT_TEST(test_scheme_related_url) { TUri url; UNIT_ASSERT_VALUES_EQUAL(url.Parse("//www.hostname.ru/path", TFeature::FeaturesRobot), TState::ParsedOK); @@ -903,120 +903,120 @@ namespace NUri { Y_UNIT_TEST_SUITE(TInvertDomainTest) { Y_UNIT_TEST(TestInvert) { - TString a; - UNIT_ASSERT_EQUAL(InvertDomain(a), ""); - TString aa(".:/foo"); - UNIT_ASSERT_EQUAL(InvertDomain(aa), ".:/foo"); - TString aaa("/foo.bar:"); - UNIT_ASSERT_EQUAL(InvertDomain(aaa), "/foo.bar:"); - TString b("ru"); - UNIT_ASSERT_EQUAL(InvertDomain(b), "ru"); - TString c(".ru"); - UNIT_ASSERT_EQUAL(InvertDomain(c), "ru."); - TString d("ru."); - UNIT_ASSERT_EQUAL(InvertDomain(d), ".ru"); - TString e("www.yandex.ru:80/yandsearch?text=foo"); - UNIT_ASSERT_EQUAL(InvertDomain(e), "ru.yandex.www:80/yandsearch?text=foo"); - TString f("www.yandex.ru:80/yandsearch?text=foo"); - InvertDomain(f.begin(), f.begin() + 10); - UNIT_ASSERT_EQUAL(f, "yandex.www.ru:80/yandsearch?text=foo"); - TString g("https://www.yandex.ru:80//"); - UNIT_ASSERT_EQUAL(InvertDomain(g), "https://ru.yandex.www:80//"); - TString h("www.yandex.ru:8080/redir.pl?url=https://google.com/"); - UNIT_ASSERT_EQUAL(InvertDomain(h), "ru.yandex.www:8080/redir.pl?url=https://google.com/"); - } + TString a; + UNIT_ASSERT_EQUAL(InvertDomain(a), ""); + TString aa(".:/foo"); + UNIT_ASSERT_EQUAL(InvertDomain(aa), ".:/foo"); + TString aaa("/foo.bar:"); + UNIT_ASSERT_EQUAL(InvertDomain(aaa), "/foo.bar:"); + TString b("ru"); + UNIT_ASSERT_EQUAL(InvertDomain(b), "ru"); + TString c(".ru"); + UNIT_ASSERT_EQUAL(InvertDomain(c), "ru."); + TString d("ru."); + UNIT_ASSERT_EQUAL(InvertDomain(d), ".ru"); + TString e("www.yandex.ru:80/yandsearch?text=foo"); + UNIT_ASSERT_EQUAL(InvertDomain(e), "ru.yandex.www:80/yandsearch?text=foo"); + TString f("www.yandex.ru:80/yandsearch?text=foo"); + InvertDomain(f.begin(), f.begin() + 10); + UNIT_ASSERT_EQUAL(f, "yandex.www.ru:80/yandsearch?text=foo"); + TString g("https://www.yandex.ru:80//"); + UNIT_ASSERT_EQUAL(InvertDomain(g), "https://ru.yandex.www:80//"); + TString h("www.yandex.ru:8080/redir.pl?url=https://google.com/"); + UNIT_ASSERT_EQUAL(InvertDomain(h), "ru.yandex.www:8080/redir.pl?url=https://google.com/"); + } } - TQueryArg::EProcessed ProcessQargs(TString url, TString& processed, TQueryArgFilter filter = 0, void* filterData = 0) { - TUri uri; - uri.Parse(url, NUri::TFeature::FeaturesRecommended); + TQueryArg::EProcessed ProcessQargs(TString url, TString& processed, TQueryArgFilter filter = 0, void* filterData = 0) { + TUri uri; + uri.Parse(url, NUri::TFeature::FeaturesRecommended); - TQueryArgProcessing processing(TQueryArg::FeatureSortByName | (filter ? TQueryArg::FeatureFilter : 0) | TQueryArg::FeatureRewriteDirty, filter, filterData); - auto result = processing.Process(uri); - processed = uri.PrintS(); - return result; - } + TQueryArgProcessing processing(TQueryArg::FeatureSortByName | (filter ? TQueryArg::FeatureFilter : 0) | TQueryArg::FeatureRewriteDirty, filter, filterData); + auto result = processing.Process(uri); + processed = uri.PrintS(); + return result; + } - TString SortQargs(TString url) { - TString r; - ProcessQargs(url, r); - return r; + TString SortQargs(TString url) { + TString r; + ProcessQargs(url, r); + return r; } - bool QueryArgsFilter(const TQueryArg& arg, void* filterData) { - const char* skipName = static_cast<const char*>(filterData); - return arg.Name != skipName; - } + bool QueryArgsFilter(const TQueryArg& arg, void* filterData) { + const char* skipName = static_cast<const char*>(filterData); + return arg.Name != skipName; + } - TString FilterQargs(TString url, const char* name) { - TString r; - ProcessQargs(url, r, &QueryArgsFilter, const_cast<char*>(name)); - return r; - } + TString FilterQargs(TString url, const char* name) { + TString r; + ProcessQargs(url, r, &QueryArgsFilter, const_cast<char*>(name)); + return r; + } Y_UNIT_TEST_SUITE(QargsTest) { Y_UNIT_TEST(TestSorting) { - UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/"), "http://ya.ru/"); - UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?"), "http://ya.ru/?"); - UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?some=value"), "http://ya.ru/?some=value"); - UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?b=1&a=2"), "http://ya.ru/?a=2&b=1"); - UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?b=1&a=2&a=3"), "http://ya.ru/?a=3&a=2&b=1"); + UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/"), "http://ya.ru/"); + UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?"), "http://ya.ru/?"); + UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?some=value"), "http://ya.ru/?some=value"); + UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?b=1&a=2"), "http://ya.ru/?a=2&b=1"); + UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?b=1&a=2&a=3"), "http://ya.ru/?a=3&a=2&b=1"); - UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?aaa=3&b=b&a=1&aa=2"), "http://ya.ru/?a=1&aa=2&aaa=3&b=b"); + UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?aaa=3&b=b&a=1&aa=2"), "http://ya.ru/?a=1&aa=2&aaa=3&b=b"); - UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?a=1&b=1&c=1"), "http://ya.ru/?a=1&b=1&c=1"); - UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?b=1&a=1&c=1"), "http://ya.ru/?a=1&b=1&c=1"); - UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?c=1&a=1&b=1"), "http://ya.ru/?a=1&b=1&c=1"); + UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?a=1&b=1&c=1"), "http://ya.ru/?a=1&b=1&c=1"); + UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?b=1&a=1&c=1"), "http://ya.ru/?a=1&b=1&c=1"); + UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?c=1&a=1&b=1"), "http://ya.ru/?a=1&b=1&c=1"); - UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?c=1&a=1&a=1&b=1&c=1&b=1"), "http://ya.ru/?a=1&a=1&b=1&b=1&c=1&c=1"); + UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?c=1&a=1&a=1&b=1&c=1&b=1"), "http://ya.ru/?a=1&a=1&b=1&b=1&c=1&c=1"); - UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?b==&a=&&c="), "http://ya.ru/?a=&b==&c="); - } + UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?b==&a=&&c="), "http://ya.ru/?a=&b==&c="); + } Y_UNIT_TEST(TestParsingCorners) { - TString s; + TString s; UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?=", s), TQueryArg::ProcessedOK); UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?some", s), TQueryArg::ProcessedOK); UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?some=", s), TQueryArg::ProcessedOK); - UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/", s), TQueryArg::ProcessedOK); - UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?&", s), TQueryArg::ProcessedOK); - UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?&&", s), TQueryArg::ProcessedOK); - UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?some=", s), TQueryArg::ProcessedOK); - UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?some==", s), TQueryArg::ProcessedOK); - UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?some=&&", s), TQueryArg::ProcessedOK); + UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/", s), TQueryArg::ProcessedOK); + UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?&", s), TQueryArg::ProcessedOK); + UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?&&", s), TQueryArg::ProcessedOK); + UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?some=", s), TQueryArg::ProcessedOK); + UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?some==", s), TQueryArg::ProcessedOK); + UNIT_ASSERT_EQUAL(ProcessQargs("http://ya.ru/?some=&&", s), TQueryArg::ProcessedOK); UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?="), "http://ya.ru/?="); UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?some=="), "http://ya.ru/?some=="); UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?&&"), "http://ya.ru/?&&"); UNIT_ASSERT_STRINGS_EQUAL(SortQargs("http://ya.ru/?a"), "http://ya.ru/?a"); - } + } Y_UNIT_TEST(TestFiltering) { - UNIT_ASSERT_STRINGS_EQUAL(FilterQargs("http://ya.ru/?some=value", "missing"), "http://ya.ru/?some=value"); - UNIT_ASSERT_STRINGS_EQUAL(FilterQargs("http://ya.ru/?b=1&a=2", "b"), "http://ya.ru/?a=2"); - UNIT_ASSERT_STRINGS_EQUAL(FilterQargs("http://ya.ru/?b=1&a=2&a=3", "a"), "http://ya.ru/?b=1"); - UNIT_ASSERT_STRINGS_EQUAL(FilterQargs("http://ya.ru/?some=&another=", "another"), "http://ya.ru/?some="); - } + UNIT_ASSERT_STRINGS_EQUAL(FilterQargs("http://ya.ru/?some=value", "missing"), "http://ya.ru/?some=value"); + UNIT_ASSERT_STRINGS_EQUAL(FilterQargs("http://ya.ru/?b=1&a=2", "b"), "http://ya.ru/?a=2"); + UNIT_ASSERT_STRINGS_EQUAL(FilterQargs("http://ya.ru/?b=1&a=2&a=3", "a"), "http://ya.ru/?b=1"); + UNIT_ASSERT_STRINGS_EQUAL(FilterQargs("http://ya.ru/?some=&another=", "another"), "http://ya.ru/?some="); + } Y_UNIT_TEST(TestRemoveEmptyFeature) { - TUri uri; - uri.Parse("http://ya.ru/?", NUri::TFeature::FeaturesRecommended); + TUri uri; + uri.Parse("http://ya.ru/?", NUri::TFeature::FeaturesRecommended); - TQueryArgProcessing processing(TQueryArg::FeatureRemoveEmptyQuery | TQueryArg::FeatureRewriteDirty); - auto result = processing.Process(uri); - UNIT_ASSERT_EQUAL(result, TQueryArg::ProcessedOK); - UNIT_ASSERT_STRINGS_EQUAL(uri.PrintS(), "http://ya.ru/"); - } + TQueryArgProcessing processing(TQueryArg::FeatureRemoveEmptyQuery | TQueryArg::FeatureRewriteDirty); + auto result = processing.Process(uri); + UNIT_ASSERT_EQUAL(result, TQueryArg::ProcessedOK); + UNIT_ASSERT_STRINGS_EQUAL(uri.PrintS(), "http://ya.ru/"); + } Y_UNIT_TEST(TestNoRemoveEmptyFeature) { - TUri uri; - uri.Parse("http://ya.ru/?", NUri::TFeature::FeaturesRecommended); - - TQueryArgProcessing processing(0); - auto result = processing.Process(uri); - UNIT_ASSERT_EQUAL(result, TQueryArg::ProcessedOK); - UNIT_ASSERT_STRINGS_EQUAL(uri.PrintS(), "http://ya.ru/?"); - } - } + TUri uri; + uri.Parse("http://ya.ru/?", NUri::TFeature::FeaturesRecommended); + + TQueryArgProcessing processing(0); + auto result = processing.Process(uri); + UNIT_ASSERT_EQUAL(result, TQueryArg::ProcessedOK); + UNIT_ASSERT_STRINGS_EQUAL(uri.PrintS(), "http://ya.ru/?"); + } + } } diff --git a/library/cpp/uri/uri_ut.h b/library/cpp/uri/uri_ut.h index f8ac6e4092..699f2d1581 100644 --- a/library/cpp/uri/uri_ut.h +++ b/library/cpp/uri/uri_ut.h @@ -5,19 +5,19 @@ #include <library/cpp/testing/unittest/registar.h> namespace NUri { - struct TTest { - TStringBuf Val; - TParseFlags Flags; - TState::EParsed State; - TStringBuf Scheme; - TStringBuf User; - TStringBuf Pass; - TStringBuf Host; - ui16 Port; - TStringBuf Path; - TStringBuf Query; - TStringBuf Frag; - }; + struct TTest { + TStringBuf Val; + TParseFlags Flags; + TState::EParsed State; + TStringBuf Scheme; + TStringBuf User; + TStringBuf Pass; + TStringBuf Host; + ui16 Port; + TStringBuf Path; + TStringBuf Query; + TStringBuf Frag; + }; } @@ -29,53 +29,53 @@ namespace NUri { UNIT_ASSERT_UNEQUAL_C(url, url2, URL_MSG(url1, url2, "==")) #define CMP_FLD(url, test, fld) \ - UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::Field##fld), test.fld) + UNIT_ASSERT_VALUES_EQUAL(url.GetField(TField::Field##fld), test.fld) -#define CMP_URL(url, test) \ - do { \ - CMP_FLD(url, test, Scheme); \ - CMP_FLD(url, test, User); \ - CMP_FLD(url, test, Pass); \ - CMP_FLD(url, test, Host); \ - UNIT_ASSERT_VALUES_EQUAL(url.GetPort(), test.Port); \ - CMP_FLD(url, test, Path); \ - CMP_FLD(url, test, Query); \ - CMP_FLD(url, test, Frag); \ - } while (false) +#define CMP_URL(url, test) \ + do { \ + CMP_FLD(url, test, Scheme); \ + CMP_FLD(url, test, User); \ + CMP_FLD(url, test, Pass); \ + CMP_FLD(url, test, Host); \ + UNIT_ASSERT_VALUES_EQUAL(url.GetPort(), test.Port); \ + CMP_FLD(url, test, Path); \ + CMP_FLD(url, test, Query); \ + CMP_FLD(url, test, Frag); \ + } while (false) -#define URL_TEST_ENC(url, test, enc) \ - do { \ - TState::EParsed st = url.ParseUri(test.Val, test.Flags, 0, enc); \ - UNIT_ASSERT_VALUES_EQUAL(st, test.State); \ - CMP_URL(url, test); \ - if (TState::ParsedOK != st) \ - break; \ - TUri _url; \ - TString urlstr, urlstr2; \ - urlstr = url.PrintS(); \ - TState::EParsed st2 = _url.ParseUri(urlstr, \ - (test.Flags & ~TFeature::FeatureNoRelPath) | TFeature::FeatureAllowRootless, 0, enc); \ - if (TState::ParsedEmpty != st2) \ - UNIT_ASSERT_VALUES_EQUAL(st2, test.State); \ - urlstr2 = _url.PrintS(); \ - UNIT_ASSERT_VALUES_EQUAL(urlstr, urlstr2); \ - CMP_URL(_url, test); \ - UNIT_ASSERT_VALUES_EQUAL(url.GetUrlFieldMask(), _url.GetUrlFieldMask()); \ - URL_EQ(url, _url); \ - const TStringBuf hostascii = url.GetField(TField::FieldHostAscii); \ - if (hostascii.Empty()) \ - break; \ - urlstr = url.PrintS(TField::FlagHostAscii); \ - st2 = _url.ParseUri(urlstr, \ - (test.Flags & ~TFeature::FeatureNoRelPath) | TFeature::FeatureAllowRootless, 0, enc); \ - UNIT_ASSERT_VALUES_EQUAL(st2, test.State); \ - urlstr2 = _url.PrintS(); \ - UNIT_ASSERT_VALUES_EQUAL(urlstr, urlstr2); \ - TTest test2 = test; \ - test2.Host = hostascii; \ - CMP_URL(_url, test2); \ - UNIT_ASSERT_VALUES_EQUAL(url.GetUrlFieldMask(), _url.GetUrlFieldMask()); \ - } while (false) +#define URL_TEST_ENC(url, test, enc) \ + do { \ + TState::EParsed st = url.ParseUri(test.Val, test.Flags, 0, enc); \ + UNIT_ASSERT_VALUES_EQUAL(st, test.State); \ + CMP_URL(url, test); \ + if (TState::ParsedOK != st) \ + break; \ + TUri _url; \ + TString urlstr, urlstr2; \ + urlstr = url.PrintS(); \ + TState::EParsed st2 = _url.ParseUri(urlstr, \ + (test.Flags & ~TFeature::FeatureNoRelPath) | TFeature::FeatureAllowRootless, 0, enc); \ + if (TState::ParsedEmpty != st2) \ + UNIT_ASSERT_VALUES_EQUAL(st2, test.State); \ + urlstr2 = _url.PrintS(); \ + UNIT_ASSERT_VALUES_EQUAL(urlstr, urlstr2); \ + CMP_URL(_url, test); \ + UNIT_ASSERT_VALUES_EQUAL(url.GetUrlFieldMask(), _url.GetUrlFieldMask()); \ + URL_EQ(url, _url); \ + const TStringBuf hostascii = url.GetField(TField::FieldHostAscii); \ + if (hostascii.Empty()) \ + break; \ + urlstr = url.PrintS(TField::FlagHostAscii); \ + st2 = _url.ParseUri(urlstr, \ + (test.Flags & ~TFeature::FeatureNoRelPath) | TFeature::FeatureAllowRootless, 0, enc); \ + UNIT_ASSERT_VALUES_EQUAL(st2, test.State); \ + urlstr2 = _url.PrintS(); \ + UNIT_ASSERT_VALUES_EQUAL(urlstr, urlstr2); \ + TTest test2 = test; \ + test2.Host = hostascii; \ + CMP_URL(_url, test2); \ + UNIT_ASSERT_VALUES_EQUAL(url.GetUrlFieldMask(), _url.GetUrlFieldMask()); \ + } while (false) #define URL_TEST(url, test) \ URL_TEST_ENC(url, test, CODES_UTF8) diff --git a/library/cpp/uri/ut/ya.make b/library/cpp/uri/ut/ya.make index b2b2c1291a..14cc4e01d0 100644 --- a/library/cpp/uri/ut/ya.make +++ b/library/cpp/uri/ut/ya.make @@ -2,10 +2,10 @@ UNITTEST_FOR(library/cpp/uri) OWNER(leo) -NO_OPTIMIZE() +NO_OPTIMIZE() NO_WSHADOW() - + PEERDIR( library/cpp/html/entity ) diff --git a/library/cpp/uri/ya.make b/library/cpp/uri/ya.make index 8fc808a6af..8cd5ef25de 100644 --- a/library/cpp/uri/ya.make +++ b/library/cpp/uri/ya.make @@ -1,10 +1,10 @@ -LIBRARY() +LIBRARY() OWNER( mvel g:base ) - + SRCS( assign.cpp common.cpp @@ -15,8 +15,8 @@ SRCS( parse.cpp qargs.cpp uri.cpp - encodefsm.rl6 - parsefsm.rl6 + encodefsm.rl6 + parsefsm.rl6 ) PEERDIR( diff --git a/library/cpp/xml/document/libxml-guards.h b/library/cpp/xml/document/libxml-guards.h index 4188cecff1..7e8a0bfece 100644 --- a/library/cpp/xml/document/libxml-guards.h +++ b/library/cpp/xml/document/libxml-guards.h @@ -1,5 +1,5 @@ #pragma once - + #include <library/cpp/xml/init/ptr.h> #include <util/generic/ptr.h> #include <libxml/xmlstring.h> @@ -47,4 +47,4 @@ namespace NXml { typedef TxmlParserCtxtPtr TParserCtxtPtr; typedef TAutoPtr<xmlSaveCtxt, NDetail::TSaveCtxtPtrTraits> TSaveCtxtPtr; -} +} diff --git a/library/cpp/xml/document/node-attr.h b/library/cpp/xml/document/node-attr.h index 6e74403943..8cdcacfed2 100644 --- a/library/cpp/xml/document/node-attr.h +++ b/library/cpp/xml/document/node-attr.h @@ -10,18 +10,18 @@ namespace NXml { // libxml defines unsigned char -> xmlChar, // and all functions use xmlChar. - inline static const char* CAST2CHAR(const xmlChar* x) { - return reinterpret_cast<const char*>(x); - } - inline static const xmlChar* XMLCHAR(const char* x) { - return reinterpret_cast<const xmlChar*>(x); - } + inline static const char* CAST2CHAR(const xmlChar* x) { + return reinterpret_cast<const char*>(x); + } + inline static const xmlChar* XMLCHAR(const char* x) { + return reinterpret_cast<const xmlChar*>(x); + } template <class T> void TNode::AttrInternal(TCharPtr& value, T& res, TStringBuf errContext) const { try { res = FromString<T>(CAST2CHAR(value.Get())); - } catch (TFromStringException&) { + } catch (TFromStringException&) { THROW(XmlException, "Failed to convert string " << TString{TStringBuf(CAST2CHAR(value.Get())).substr(0, 50)}.Quote() << " from '" << errContext << "' to requested type"); } } @@ -35,7 +35,7 @@ namespace NXml { template <class T> T TNode::Attr(TZtStringBuf name) const { TCharPtr value(xmlGetProp(NodePointer, XMLCHAR(name.c_str()))); - if (!value) { + if (!value) { THROW(AttributeNotFound, Path() << "@" << name); } @@ -47,7 +47,7 @@ namespace NXml { template <class T> T TNode::Attr(TZtStringBuf name, const T& defvalue) const { TCharPtr attr(xmlGetProp(NodePointer, XMLCHAR(name.c_str()))); - if (!attr) { + if (!attr) { return defvalue; } @@ -59,7 +59,7 @@ namespace NXml { template <class T> void TNode::Attr(TZtStringBuf name, T& value) const { TCharPtr attr(xmlGetProp(NodePointer, XMLCHAR(name.c_str()))); - if (!attr) { + if (!attr) { THROW(AttributeNotFound, Path() << name); } @@ -70,16 +70,16 @@ namespace NXml { void TNode::Attr(TZtStringBuf name, T& value, const T& defvalue) const { TCharPtr attr(xmlGetProp(NodePointer, XMLCHAR(name.c_str()))); - if (!attr) { + if (!attr) { value = defvalue; - } else { + } else { AttrInternal(attr, value, name); } } template <class T> - T TNode::Value() const { - if (!NodePointer || xmlIsBlankNode(NodePointer)) { + T TNode::Value() const { + if (!NodePointer || xmlIsBlankNode(NodePointer)) { THROW(NodeIsBlank, Path()); } @@ -90,8 +90,8 @@ namespace NXml { } template <class T> - T TNode::Value(const T& defvalue) const { - if (!NodePointer || xmlIsBlankNode(NodePointer)) { + T TNode::Value(const T& defvalue) const { + if (!NodePointer || xmlIsBlankNode(NodePointer)) { return defvalue; } @@ -115,14 +115,14 @@ namespace NXml { } inline void TNode::SetAttr(TZtStringBuf name, TZtStringBuf value) { - xmlAttr* attr = xmlSetProp(NodePointer, XMLCHAR(name.c_str()), XMLCHAR(value.c_str())); - - if (!attr) { - THROW(XmlException, "Can't set node attribute <" - << name - << "> to <" - << value - << ">"); + xmlAttr* attr = xmlSetProp(NodePointer, XMLCHAR(name.c_str()), XMLCHAR(value.c_str())); + + if (!attr) { + THROW(XmlException, "Can't set node attribute <" + << name + << "> to <" + << value + << ">"); } } @@ -135,20 +135,20 @@ namespace NXml { } inline void TNode::SetAttr(TZtStringBuf name) { - xmlAttr* attr = xmlSetProp(NodePointer, XMLCHAR(name.c_str()), nullptr); + xmlAttr* attr = xmlSetProp(NodePointer, XMLCHAR(name.c_str()), nullptr); - if (!attr) { - THROW(XmlException, "Can't set node empty attribute <" - << name - << ">"); + if (!attr) { + THROW(XmlException, "Can't set node empty attribute <" + << name + << ">"); } } inline void TNode::DelAttr(TZtStringBuf name) { if (xmlUnsetProp(NodePointer, XMLCHAR(name.c_str())) < 0) - THROW(XmlException, "Can't delete node attribute <" - << name - << ">"); + THROW(XmlException, "Can't delete node attribute <" + << name + << ">"); } template <class T> @@ -160,23 +160,23 @@ namespace NXml { } inline TNode TNode::AddChild(TZtStringBuf name, TZtStringBuf value) { - if (IsNull()) { - THROW(XmlException, "addChild [name=" << name << ", value=" << value - << "]: can't add child to null node"); + if (IsNull()) { + THROW(XmlException, "addChild [name=" << name << ", value=" << value + << "]: can't add child to null node"); } - xmlNode* child = nullptr; + xmlNode* child = nullptr; - if (value.empty()) { + if (value.empty()) { child = xmlNewTextChild(NodePointer, nullptr, XMLCHAR(name.c_str()), nullptr); - } else { + } else { child = xmlNewTextChild( NodePointer, nullptr, XMLCHAR(name.c_str()), XMLCHAR(value.c_str())); } - if (!child) { - THROW(XmlException, "addChild [name=" << name << ", value=" << value - << "]: xmlNewTextChild returned NULL"); + if (!child) { + THROW(XmlException, "addChild [name=" << name << ", value=" << value + << "]: xmlNewTextChild returned NULL"); } return TNode(DocPointer, child); @@ -191,19 +191,19 @@ namespace NXml { } inline TNode TNode::AddText(TStringBuf value) { - if (IsNull()) { - THROW(XmlException, "addChild [value=" << value - << "]: can't add child to null node"); + if (IsNull()) { + THROW(XmlException, "addChild [value=" << value + << "]: can't add child to null node"); } xmlNode* child = xmlNewTextLen((xmlChar*)value.data(), value.size()); child = xmlAddChild(NodePointer, child); - if (!child) { - THROW(XmlException, "addChild [value=" << value - << "]: xmlNewTextChild returned NULL"); + if (!child) { + THROW(XmlException, "addChild [value=" << value + << "]: xmlNewTextChild returned NULL"); } return TNode(DocPointer, child); } -} +} diff --git a/library/cpp/xml/document/xml-document-decl.h b/library/cpp/xml/document/xml-document-decl.h index bfda1fb7e6..5461738d8c 100644 --- a/library/cpp/xml/document/xml-document-decl.h +++ b/library/cpp/xml/document/xml-document-decl.h @@ -1,5 +1,5 @@ #pragma once - + #include <library/cpp/string_utils/ztstrbuf/ztstrbuf.h> #include <util/generic/string.h> @@ -17,9 +17,9 @@ namespace NXml { using TXPathContext = xmlXPathContext; - class TDocument { + class TDocument { public: - enum Source { + enum Source { File, String, RootName, @@ -62,11 +62,11 @@ namespace NXml { return s.Str(); } - void Swap(TDocument& that) { + void Swap(TDocument& that) { std::swap(this->Doc, that.Doc); } - xmlDocPtr GetImpl() { + xmlDocPtr GetImpl() { return Doc.Get(); } @@ -76,8 +76,8 @@ namespace NXml { TDocument(TDocHolder doc) : Doc(std::move(doc)) - { - } + { + } TDocHolder Doc; }; @@ -88,22 +88,22 @@ namespace NXml { }; typedef TVector<TNamespaceForXPath> TNamespacesForXPath; - class TConstNodes { + class TConstNodes { private: - struct TConstNodesRef { - explicit TConstNodesRef(TConstNodes& n) - : r_(n) - { - } + struct TConstNodesRef { + explicit TConstNodesRef(TConstNodes& n) + : r_(n) + { + } TConstNodes& r_; }; - + public: TConstNodes(const TConstNodes& nodes); TConstNodes& operator=(const TConstNodes& nodes); TConstNodes(TConstNodesRef ref); - TConstNodes& operator=(TConstNodesRef ref); + TConstNodes& operator=(TConstNodesRef ref); operator TConstNodesRef(); @@ -116,17 +116,17 @@ namespace NXml { /** * get number of nodes */ - size_t Size() const { - return SizeValue; - } - size_t size() const { - return SizeValue; - } + size_t Size() const { + return SizeValue; + } + size_t size() const { + return SizeValue; + } struct TNodeIter { const TConstNodes& Nodes; size_t Index; - TConstNode operator*() const; + TConstNode operator*() const; bool operator==(const TNodeIter& other) const { return Index == other.Index; } @@ -138,12 +138,12 @@ namespace NXml { return *this; } }; - TNodeIter begin() const { - return TNodeIter{*this, 0}; - } - TNodeIter end() const { - return TNodeIter{*this, size()}; - } + TNodeIter begin() const { + return TNodeIter{*this, 0}; + } + TNodeIter end() const { + return TNodeIter{*this, size()}; + } private: friend class TDocument; @@ -157,7 +157,7 @@ namespace NXml { TXPathObjectPtr Obj; }; - class TNode { + class TNode { public: friend class TDocument; friend class TConstNode; @@ -169,11 +169,11 @@ namespace NXml { bool IsNull() const; /** - * check if node is element node - */ - bool IsElementNode() const; - - /** + * check if node is element node + */ + bool IsElementNode() const; + + /** * Create xpath context to be used later for fast xpath evaluation. * @param nss: explicitly specify XML namespaces to use and their prefixes * @@ -259,9 +259,9 @@ namespace NXml { TNode FirstChild(TZtStringBuf name); TConstNode FirstChild(TZtStringBuf name) const; - TNode FirstChild(); - TConstNode FirstChild() const; - + TNode FirstChild(); + TConstNode FirstChild() const; + /** * get parent node * throws exception if has no parent @@ -278,9 +278,9 @@ namespace NXml { TNode NextSibling(TZtStringBuf name); TConstNode NextSibling(TZtStringBuf name) const; - TNode NextSibling(); - TConstNode NextSibling() const; - + TNode NextSibling(); + TConstNode NextSibling() const; + /** * create child node * @param name: child name @@ -371,7 +371,7 @@ namespace NXml { * set node value * @param value: new text value */ - template <class T> + template <class T> typename std::enable_if<!std::is_convertible_v<T, TStringBuf>, void>::type SetValue(const T& value); @@ -451,8 +451,8 @@ namespace NXml { TNode() : NodePointer(nullptr) , DocPointer(nullptr) - { - } + { + } private: friend class TConstNodes; @@ -460,12 +460,12 @@ namespace NXml { TNode(xmlDoc* doc, xmlNode* node) : NodePointer(node) , DocPointer(doc) - { - } + { + } TNode Find(xmlNode* start, TZtStringBuf name); - template <class T> + template <class T> void AttrInternal(TCharPtr& value, T& res, TStringBuf errContext) const; void SaveInternal(IOutputStream& stream, TZtStringBuf enc, int options) const; @@ -474,7 +474,7 @@ namespace NXml { xmlDoc* DocPointer; }; - class TConstNode { + class TConstNode { public: friend class TDocument; friend class TConstNodes; @@ -482,15 +482,15 @@ namespace NXml { /** * check if node is null */ - bool IsNull() const { + bool IsNull() const { return ActualNode.IsNull(); } - bool IsElementNode() const { - return ActualNode.IsElementNode(); - } - - TConstNode Parent() const { + bool IsElementNode() const { + return ActualNode.IsElementNode(); + } + + TConstNode Parent() const { return ActualNode.Parent(); } @@ -498,7 +498,7 @@ namespace NXml { * Create xpath context to be used later for fast xpath evaluation. * @param nss: explicitly specify XML namespaces to use and their prefixes */ - TXPathContextPtr CreateXPathContext(const TNamespacesForXPath& nss = TNamespacesForXPath()) const { + TXPathContextPtr CreateXPathContext(const TNamespacesForXPath& nss = TNamespacesForXPath()) const { return ActualNode.CreateXPathContext(nss); } @@ -580,10 +580,10 @@ namespace NXml { return ActualNode.FirstChild(name); } - TConstNode FirstChild() const { - return ActualNode.FirstChild(); - } - + TConstNode FirstChild() const { + return ActualNode.FirstChild(); + } + /** * get node neighbour * @param name: neighbour name @@ -593,10 +593,10 @@ namespace NXml { return ActualNode.NextSibling(name); } - TConstNode NextSibling() const { - return ActualNode.NextSibling(); - } - + TConstNode NextSibling() const { + return ActualNode.NextSibling(); + } + /** * get node attribute * @param name: attribute name @@ -645,7 +645,7 @@ namespace NXml { * @throws exception if node is blank */ template <class T> - T Value() const { + T Value() const { return ActualNode.Value<T>(); } @@ -655,7 +655,7 @@ namespace NXml { * returns default value if node is blank */ template <class T> - T Value(const T& defvalue) const { + T Value(const T& defvalue) const { return ActualNode.Value(defvalue); } @@ -669,28 +669,28 @@ namespace NXml { /** * @return application data pointer, passed by SetPrivate */ - void* GetPrivate() const { + void* GetPrivate() const { return ActualNode.GetPrivate(); } /** * get pointer to internal node */ - const xmlNode* GetPtr() const { + const xmlNode* GetPtr() const { return ActualNode.GetPtr(); } /** * check if node is text-only node */ - bool IsText() const { + bool IsText() const { return ActualNode.IsText(); } /** * get node xpath */ - TString Path() const { + TString Path() const { return ActualNode.Path(); } @@ -704,10 +704,10 @@ namespace NXml { TConstNode() = default; TConstNode(TNode node) : ActualNode(node) - { - } + { + } - TNode ConstCast() const { + TNode ConstCast() const { return ActualNode; } @@ -715,4 +715,4 @@ namespace NXml { TNode ActualNode; }; -} +} diff --git a/library/cpp/xml/document/xml-document.cpp b/library/cpp/xml/document/xml-document.cpp index 18a554d732..679f4814af 100644 --- a/library/cpp/xml/document/xml-document.cpp +++ b/library/cpp/xml/document/xml-document.cpp @@ -4,28 +4,28 @@ #include <libxml/xpathInternals.h> #include <library/cpp/xml/init/init.h> - -#include <util/generic/yexception.h> -#include <util/folder/dirut.h> - -namespace { + +#include <util/generic/yexception.h> +#include <util/folder/dirut.h> + +namespace { struct TInit { - inline TInit() { - NXml::InitEngine(); - } - } initer; -} - + inline TInit() { + NXml::InitEngine(); + } + } initer; +} + namespace NXml { - TDocument::TDocument(const TString& xml, Source type) { - switch (type) { - case File: - ParseFile(xml); - break; - case String: - ParseString(xml); - break; - case RootName: { + TDocument::TDocument(const TString& xml, Source type) { + switch (type) { + case File: + ParseFile(xml); + break; + case String: + ParseString(xml); + break; + case RootName: { TDocHolder doc(xmlNewDoc(XMLCHAR("1.0"))); if (!doc) THROW(XmlException, "Can't create xml document."); @@ -37,16 +37,16 @@ namespace NXml { xmlDocSetRootElement(doc.Get(), node.Get()); Y_UNUSED(node.Release()); Doc = std::move(doc); - } break; - default: - THROW(InvalidArgument, "Wrong source type"); + } break; + default: + THROW(InvalidArgument, "Wrong source type"); } } TDocument::TDocument(TDocument&& doc) : Doc(std::move(doc.Doc)) - { - } + { + } TDocument& TDocument::operator=(TDocument&& doc) { if (this != &doc) @@ -55,7 +55,7 @@ namespace NXml { return *this; } - void TDocument::ParseFile(const TString& file) { + void TDocument::ParseFile(const TString& file) { if (!NFs::Exists(file)) THROW(XmlException, "File " << file << " doesn't exist"); @@ -88,7 +88,7 @@ namespace NXml { Doc = std::move(doc); } - TNode TDocument::Root() { + TNode TDocument::Root() { xmlNode* r = xmlDocGetRootElement(Doc.Get()); if (r == nullptr) THROW(XmlException, "TDocument hasn't root element"); @@ -96,7 +96,7 @@ namespace NXml { return TNode(Doc.Get(), r); } - TConstNode TDocument::Root() const { + TConstNode TDocument::Root() const { xmlNode* r = xmlDocGetRootElement(Doc.Get()); if (r == nullptr) THROW(XmlException, "TDocument hasn't root element"); @@ -104,15 +104,15 @@ namespace NXml { return TConstNode(TNode(Doc.Get(), r)); } - bool TNode::IsNull() const { + bool TNode::IsNull() const { return NodePointer == nullptr; } - bool TNode::IsElementNode() const { - return !IsNull() && (NodePointer->type == XML_ELEMENT_NODE); - } - - TXPathContextPtr TNode::CreateXPathContext(const TNamespacesForXPath& nss) const { + bool TNode::IsElementNode() const { + return !IsNull() && (NodePointer->type == XML_ELEMENT_NODE); + } + + TXPathContextPtr TNode::CreateXPathContext(const TNamespacesForXPath& nss) const { TXPathContextPtr ctx = xmlXPathNewContext(DocPointer); if (!ctx) THROW(XmlException, "Can't create empty xpath context"); @@ -196,25 +196,25 @@ namespace NXml { return const_cast<TNode*>(this)->FirstChild(name); } - TNode TNode::FirstChild() { - if (IsNull()) - THROW(XmlException, "Node is null"); - - return TNode(DocPointer, NodePointer->children); - } - - TConstNode TNode::FirstChild() const { - return const_cast<TNode*>(this)->FirstChild(); - } - - TNode TNode::Parent() { + TNode TNode::FirstChild() { + if (IsNull()) + THROW(XmlException, "Node is null"); + + return TNode(DocPointer, NodePointer->children); + } + + TConstNode TNode::FirstChild() const { + return const_cast<TNode*>(this)->FirstChild(); + } + + TNode TNode::Parent() { if (nullptr == NodePointer->parent) THROW(XmlException, "Parent node not exists"); - + return TNode(DocPointer, NodePointer->parent); } - TConstNode TNode::Parent() const { + TConstNode TNode::Parent() const { return const_cast<TNode*>(this)->Parent(); } @@ -229,17 +229,17 @@ namespace NXml { return const_cast<TNode*>(this)->NextSibling(name); } - TNode TNode::NextSibling() { - if (IsNull()) - THROW(XmlException, "Node is null"); - - return TNode(DocPointer, NodePointer->next); - } - - TConstNode TNode::NextSibling() const { - return const_cast<TNode*>(this)->NextSibling(); - } - + TNode TNode::NextSibling() { + if (IsNull()) + THROW(XmlException, "Node is null"); + + return TNode(DocPointer, NodePointer->next); + } + + TConstNode TNode::NextSibling() const { + return const_cast<TNode*>(this)->NextSibling(); + } + /* NOTE: by default child will inherit it's parent ns */ TNode TNode::AddChild(TZtStringBuf name) { @@ -254,17 +254,17 @@ namespace NXml { return TNode(DocPointer, copy); } - void TNode::SetPrivate(void* priv) { + void TNode::SetPrivate(void* priv) { NodePointer->_private = priv; } - void* TNode::GetPrivate() const { + void* TNode::GetPrivate() const { return NodePointer->_private; } TNode TNode::Find(xmlNode* start, TZtStringBuf name) { for (; start; start = start->next) - if (start->type == XML_ELEMENT_NODE && (name.empty() || !xmlStrcmp(start->name, XMLCHAR(name.c_str())))) + if (start->type == XML_ELEMENT_NODE && (name.empty() || !xmlStrcmp(start->name, XMLCHAR(name.c_str())))) return TNode(DocPointer, start); return TNode(); @@ -277,7 +277,7 @@ namespace NXml { return CAST2CHAR(NodePointer->name); } - TString TNode::Path() const { + TString TNode::Path() const { TCharPtr path(xmlGetNodePath(NodePointer)); if (!!path) return CAST2CHAR(path.Get()); @@ -285,15 +285,15 @@ namespace NXml { return ""; } - xmlNode* TNode::GetPtr() { + xmlNode* TNode::GetPtr() { return NodePointer; } - const xmlNode* TNode::GetPtr() const { + const xmlNode* TNode::GetPtr() const { return NodePointer; } - bool TNode::IsText() const { + bool TNode::IsText() const { if (IsNull()) THROW(XmlException, "Node is null"); @@ -306,7 +306,7 @@ namespace NXml { xmlFreeNode(nodePtr); } - static int XmlWriteToOstream(void* context, const char* buffer, int len) { + static int XmlWriteToOstream(void* context, const char* buffer, int len) { // possibly use to save doc as well IOutputStream* out = (IOutputStream*)context; out->Write(buffer, len); @@ -339,7 +339,7 @@ namespace NXml { } TConstNodes& TConstNodes::operator=(const TConstNodes& nodes) { - if (this != &nodes) { + if (this != &nodes) { SizeValue = nodes.Size(); Doc = nodes.Doc; Obj = nodes.Obj; @@ -355,8 +355,8 @@ namespace NXml { { } - TConstNodes& TConstNodes::operator=(TConstNodesRef ref) { - if (this != &ref.r_) { + TConstNodes& TConstNodes::operator=(TConstNodesRef ref) { + if (this != &ref.r_) { SizeValue = ref.r_.Size(); Doc = ref.r_.Doc; Obj = ref.r_.Obj; @@ -364,7 +364,7 @@ namespace NXml { return *this; } - TConstNodes::operator TConstNodesRef() { + TConstNodes::operator TConstNodesRef() { return TConstNodesRef(*this); } @@ -375,7 +375,7 @@ namespace NXml { { } - TConstNode TConstNodes::operator[](size_t number) const { + TConstNode TConstNodes::operator[](size_t number) const { if (number + 1 > Size()) THROW(XmlException, "index out of range " << number); @@ -386,8 +386,8 @@ namespace NXml { return TNode(Doc, node); } - TConstNode TConstNodes::TNodeIter::operator*() const { + TConstNode TConstNodes::TNodeIter::operator*() const { return Nodes[Index]; } -} +} diff --git a/library/cpp/xml/document/xml-document_ut.cpp b/library/cpp/xml/document/xml-document_ut.cpp index 9f537b75c4..86c995b021 100644 --- a/library/cpp/xml/document/xml-document_ut.cpp +++ b/library/cpp/xml/document/xml-document_ut.cpp @@ -5,27 +5,27 @@ Y_UNIT_TEST_SUITE(TestXmlDocument) { Y_UNIT_TEST(Iteration) { - NXml::TDocument xml( - "<?xml version=\"1.0\"?>\n" - "<root>qq<a><b></b></a>ww<c></c></root>", - NXml::TDocument::String); - - NXml::TConstNode root = xml.Root(); - UNIT_ASSERT_EQUAL(root.Name(), "root"); - NXml::TConstNode n = root.FirstChild().NextSibling(); - UNIT_ASSERT_EQUAL(n.Name(), "a"); - n = n.NextSibling().NextSibling(); - UNIT_ASSERT_EQUAL(n.Name(), "c"); - } - + NXml::TDocument xml( + "<?xml version=\"1.0\"?>\n" + "<root>qq<a><b></b></a>ww<c></c></root>", + NXml::TDocument::String); + + NXml::TConstNode root = xml.Root(); + UNIT_ASSERT_EQUAL(root.Name(), "root"); + NXml::TConstNode n = root.FirstChild().NextSibling(); + UNIT_ASSERT_EQUAL(n.Name(), "a"); + n = n.NextSibling().NextSibling(); + UNIT_ASSERT_EQUAL(n.Name(), "c"); + } + Y_UNIT_TEST(ParseString) { NXml::TDocument xml( - "<?xml version=\"1.0\"?>\n" - "<root>\n" - "<a><b len=\"15\" correct=\"1\">hello world</b></a>\n" - "<text>Некоторый текст</text>\n" - "</root>", - NXml::TDocument::String); + "<?xml version=\"1.0\"?>\n" + "<root>\n" + "<a><b len=\"15\" correct=\"1\">hello world</b></a>\n" + "<text>Некоторый текст</text>\n" + "</root>", + NXml::TDocument::String); NXml::TConstNode root = xml.Root(); NXml::TConstNode b = root.Node("a/b"); @@ -49,29 +49,29 @@ Y_UNIT_TEST_SUITE(TestXmlDocument) { TString data = xml.ToString("utf-8"); UNIT_ASSERT_EQUAL(data, "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" "<frob xyzzy=\"Frobozz\" kulness=\"0.3\" timelimit=\"3\">\n" - " <authors>\n" - " <graham name=\"Nelson\"/>\n" - " <zarf>Andrew Plotkin</zarf>\n" + " <authors>\n" + " <graham name=\"Nelson\"/>\n" + " <zarf>Andrew Plotkin</zarf>\n" " <emshort>Emily Short</emshort>\n" - " </authors>\n" - "</frob>\n"); + " </authors>\n" + "</frob>\n"); // check default utf8 output with ru { NXml::TDocument xml2("frob", NXml::TDocument::RootName); xml2.Root().SetAttr("xyzzy", "привет =)"); UNIT_ASSERT_VALUES_EQUAL(xml2.ToString(), "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" - "<frob xyzzy=\"привет =)\"/>\n"); + "<frob xyzzy=\"привет =)\"/>\n"); } } Y_UNIT_TEST(XPathNs) { using namespace NXml; TDocument xml( - "<?xml version=\"1.0\"?>\n" - "<root xmlns='http://hello.com/hello'>\n" - "<a><b len=\"15\" correct=\"1\">hello world</b></a>\n" - "<text>Некоторый текст</text>\n" - "</root>", - TDocument::String); + "<?xml version=\"1.0\"?>\n" + "<root xmlns='http://hello.com/hello'>\n" + "<a><b len=\"15\" correct=\"1\">hello world</b></a>\n" + "<text>Некоторый текст</text>\n" + "</root>", + TDocument::String); TNamespacesForXPath nss; TNamespaceForXPath ns = {"h", "http://hello.com/hello"}; @@ -115,7 +115,7 @@ Y_UNIT_TEST_SUITE(TestXmlDocument) { node = node.NextSibling(); UNIT_ASSERT_EQUAL(node.IsNull(), true); TStringStream iterLog; - for (const auto& node2 : root.Nodes("/root/*")) { + for (const auto& node2 : root.Nodes("/root/*")) { iterLog << node2.Name() << ';'; } UNIT_ASSERT_STRINGS_EQUAL(iterLog.Str(), "a;c;"); @@ -154,15 +154,15 @@ Y_UNIT_TEST_SUITE(TestXmlDocument) { Y_UNIT_TEST(DefVal) { using namespace NXml; TDocument xml("<?xml version=\"1.0\"?>\n" - "<root><a></a></root>", - NXml::TDocument::String); + "<root><a></a></root>", + NXml::TDocument::String); UNIT_ASSERT_EQUAL(xml.Root().Node("a", true).Node("b", true).Value<int>(3), 3); } Y_UNIT_TEST(NodesVsXPath) { using namespace NXml; TDocument xml("<?xml version=\"1.0\"?>\n" - "<root><a x=\"y\"></a></root>", - NXml::TDocument::String); + "<root><a x=\"y\"></a></root>", + NXml::TDocument::String); UNIT_ASSERT_EXCEPTION(xml.Root().Nodes("/root/a/@x"), yexception); UNIT_ASSERT_VALUES_EQUAL(xml.Root().XPath("/root/a/@x").Size(), 1); } @@ -170,8 +170,8 @@ Y_UNIT_TEST_SUITE(TestXmlDocument) { using namespace NXml; TDocument xml("<?xml version=\"1.0\"?>\n" "<root><a x=\"y\">first</a>" - "<a>second</a></root>", - NXml::TDocument::String); + "<a>second</a></root>", + NXml::TDocument::String); UNIT_ASSERT_EXCEPTION(xml.Root().Node("/root/a/@x"), yexception); UNIT_ASSERT_STRINGS_EQUAL(xml.Root().Node("/root/a").Value<TString>(), "first"); } @@ -185,12 +185,12 @@ Y_UNIT_TEST_SUITE(TestXmlDocument) { // do copy TDocument xml("<?xml version=\"1.0\"?>\n" - "<root><a></a></root>", - TDocument::String); + "<root><a></a></root>", + TDocument::String); TDocument xml2("<?xml version=\"1.0\"?>\n" - "<root><node><b>bold</b><i>ita</i></node></root>", - TDocument::String); + "<root><node><b>bold</b><i>ita</i></node></root>", + TDocument::String); TNode node = xml2.Root().Node("//node"); TNode place = xml.Root().Node("//a"); @@ -200,8 +200,8 @@ Y_UNIT_TEST_SUITE(TestXmlDocument) { TStringStream s; xml.Save(s, "", false); UNIT_ASSERT_VALUES_EQUAL(s.Str(), - "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" - "<root><a><node><b>bold</b><i>ita</i></node></a></root>\n"); + "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + "<root><a><node><b>bold</b><i>ita</i></node></a></root>\n"); } Y_UNIT_TEST(RenderNode) { @@ -213,8 +213,8 @@ Y_UNIT_TEST_SUITE(TestXmlDocument) { "<root>\n" "<a><b len=\"15\" correct=\"1\">hello world</b></a>\n" "<text>Некоторый текст</text>\n" - "</root>", - TDocument::String); + "</root>", + TDocument::String); TNode n = xml.Root().Node("//a"); UNIT_ASSERT_VALUES_EQUAL(n.ToString(), "<a><b len=\"15\" correct=\"1\">hello world</b></a>"); } @@ -225,8 +225,8 @@ Y_UNIT_TEST_SUITE(TestXmlDocument) { "<root xmlns='http://hello.com/hello'>\n" "<a><b len=\"15\" correct=\"1\">hello world</b></a>\n" "<text>Некоторый текст</text>\n" - "</root>", - TDocument::String); + "</root>", + TDocument::String); TNamespacesForXPath nss; TNamespaceForXPath ns = {"h", "http://hello.com/hello"}; nss.push_back(ns); diff --git a/library/cpp/xml/document/xml-options.h b/library/cpp/xml/document/xml-options.h index bb07da0cfb..5aad891d68 100644 --- a/library/cpp/xml/document/xml-options.h +++ b/library/cpp/xml/document/xml-options.h @@ -64,4 +64,4 @@ namespace NXml { int Mask; }; -} +} diff --git a/library/cpp/xml/document/xml-textreader.cpp b/library/cpp/xml/document/xml-textreader.cpp index b946f1fbf2..944d61928d 100644 --- a/library/cpp/xml/document/xml-textreader.cpp +++ b/library/cpp/xml/document/xml-textreader.cpp @@ -315,4 +315,4 @@ namespace NXml { xmlFreeTextReader(handle); } }; -} +} diff --git a/library/cpp/xml/document/xml-textreader.h b/library/cpp/xml/document/xml-textreader.h index ab4c329d26..9230456479 100644 --- a/library/cpp/xml/document/xml-textreader.h +++ b/library/cpp/xml/document/xml-textreader.h @@ -322,4 +322,4 @@ namespace NXml { THolder<xmlTextReader, TDeleter> Impl; }; -} +} diff --git a/library/cpp/xml/document/xml-textreader_ut.cpp b/library/cpp/xml/document/xml-textreader_ut.cpp index 6232dfe47e..2cd0be96bc 100644 --- a/library/cpp/xml/document/xml-textreader_ut.cpp +++ b/library/cpp/xml/document/xml-textreader_ut.cpp @@ -20,26 +20,26 @@ namespace { while (reader.Read()) { if (reader.GetNodeType() == NXml::TTextReader::ENodeType::Element && reader.GetLocalName() == localName && - reader.GetNamespaceUri() == namespaceUri) - { + reader.GetNamespaceUri() == namespaceUri) + { const NXml::TConstNode node = reader.Expand(); nodeHandlerFunc(node); } } } -} +} Y_UNIT_TEST_SUITE(TestXmlTextReader) { Y_UNIT_TEST(BasicExample) { const TString xml = "<?xml version=\"1.0\"?>\n" - "<example toto=\"1\">\n" - " <examplechild id=\"1\">\n" - " <child_of_child/>\n" - " </examplechild>\n" - " <examplechild id=\"2\" toto=\"3\">\n" - " <child_of_child>Some content : -)</child_of_child>\n" - " </examplechild>\n" - "</example>\n"; + "<example toto=\"1\">\n" + " <examplechild id=\"1\">\n" + " <child_of_child/>\n" + " </examplechild>\n" + " <examplechild id=\"2\" toto=\"3\">\n" + " <child_of_child>Some content : -)</child_of_child>\n" + " </examplechild>\n" + "</example>\n"; TStringInput input(xml); NXml::TTextReader reader(input); @@ -94,7 +94,7 @@ Y_UNIT_TEST_SUITE(TestXmlTextReader) { TItem{2, ENT::SignificantWhitespace, "#text", "", "\n "}, TItem{1, ENT::EndElement, "examplechild", "id: 2, toto: 3", ""}, TItem{1, ENT::SignificantWhitespace, "#text", "", "\n"}, - TItem{0, ENT::EndElement, "example", "toto: 1", ""}}; + TItem{0, ENT::EndElement, "example", "toto: 1", ""}}; UNIT_ASSERT_VALUES_EQUAL(found.size(), expected.size()); @@ -108,31 +108,31 @@ Y_UNIT_TEST_SUITE(TestXmlTextReader) { } const TString GEODATA = "<?xml version=\"1.0\" encoding=\"utf-8\"?>" - "<root>" - "" - " <country id=\"225\">" - " <name>Россия</name>" - " <cities>" - " <city>Москва</city>" - " <city>Санкт-Петербург</city>" - " </cities>" - " </country>" - "" - " <country id=\"149\">" - " <name>Беларусь</name>" - " <cities>" - " <city>Минск</city>" - " </cities>" - " </country>" - "" - " <country id=\"187\">" - " <name>Украина</name>" - " <cities>" - " <city>Киев</city>" - " </cities>" - " </country>" - "" - "</root>"; + "<root>" + "" + " <country id=\"225\">" + " <name>Россия</name>" + " <cities>" + " <city>Москва</city>" + " <city>Санкт-Петербург</city>" + " </cities>" + " </country>" + "" + " <country id=\"149\">" + " <name>Беларусь</name>" + " <cities>" + " <city>Минск</city>" + " </cities>" + " </country>" + "" + " <country id=\"187\">" + " <name>Украина</name>" + " <cities>" + " <city>Киев</city>" + " </cities>" + " </country>" + "" + "</root>"; Y_UNIT_TEST(ParseXmlSimple) { struct TCountry { @@ -253,10 +253,10 @@ Y_UNIT_TEST_SUITE(TestXmlTextReader) { Y_UNIT_TEST(NamespaceHell) { using TNS = NXml::TNamespaceForXPath; - const NXml::TNamespacesForXPath ns = { + const NXml::TNamespacesForXPath ns = { TNS{"b", "http://maps.yandex.ru/backa/1.x"}, TNS{"gml", "http://www.opengis.net/gml"}, - TNS{"xal", "urn:oasis:names:tc:ciq:xsdschema:xAL:2.0"}}; + TNS{"xal", "urn:oasis:names:tc:ciq:xsdschema:xAL:2.0"}}; int count = 0; THashMap<TString, TString> positions; diff --git a/library/cpp/xml/document/ya.make b/library/cpp/xml/document/ya.make index 86bbd639cf..9fed445b29 100644 --- a/library/cpp/xml/document/ya.make +++ b/library/cpp/xml/document/ya.make @@ -1,7 +1,7 @@ -LIBRARY() - -OWNER(finder) +LIBRARY() +OWNER(finder) + SRCS( xml-document.cpp xml-textreader.cpp diff --git a/library/cpp/xml/init/init.cpp b/library/cpp/xml/init/init.cpp index aa96c2dd31..9ecdb974d8 100644 --- a/library/cpp/xml/init/init.cpp +++ b/library/cpp/xml/init/init.cpp @@ -1,39 +1,39 @@ -#include "init.h" - +#include "init.h" + #include <libxml/xmlIO.h> #include <libxml/parser.h> #include <libxml/parserInternals.h> #include <libxml/tree.h> - + #include <library/cpp/charset/recyr.hh> -#include <util/generic/singleton.h> - -namespace { +#include <util/generic/singleton.h> + +namespace { int CharEncodingInput(unsigned char* out, int* outlen, const unsigned char* in, int* inlen) { - size_t read = 0, written = 0; - RECODE_RESULT r = Recode(CODES_WIN, CODES_UTF8, (const char*)in, (char*)out, (size_t)*inlen, (size_t)*outlen, read, written); - if (r == RECODE_EOOUTPUT) - return -1; - if (r != RECODE_OK) - return -2; - *inlen = (int)read; - *outlen = (int)written; - return (int)written; - } - - class TLibXml2 { - public: - inline TLibXml2() { - xmlInitParser(); + size_t read = 0, written = 0; + RECODE_RESULT r = Recode(CODES_WIN, CODES_UTF8, (const char*)in, (char*)out, (size_t)*inlen, (size_t)*outlen, read, written); + if (r == RECODE_EOOUTPUT) + return -1; + if (r != RECODE_OK) + return -2; + *inlen = (int)read; + *outlen = (int)written; + return (int)written; + } + + class TLibXml2 { + public: + inline TLibXml2() { + xmlInitParser(); xmlNewCharEncodingHandler("windows-1251", CharEncodingInput, nullptr); - } - + } + inline ~TLibXml2() { - xmlCleanupParser(); - } - }; -} - -void NXml::InitEngine() { - Singleton<TLibXml2>(); -} + xmlCleanupParser(); + } + }; +} + +void NXml::InitEngine() { + Singleton<TLibXml2>(); +} diff --git a/library/cpp/xml/init/init.h b/library/cpp/xml/init/init.h index 2cdc46c655..cf3c8bd743 100644 --- a/library/cpp/xml/init/init.h +++ b/library/cpp/xml/init/init.h @@ -1,5 +1,5 @@ -#pragma once - -namespace NXml { - void InitEngine(); -} +#pragma once + +namespace NXml { + void InitEngine(); +} diff --git a/library/cpp/xml/init/ptr.cpp b/library/cpp/xml/init/ptr.cpp index e11a6460c3..54447188ef 100644 --- a/library/cpp/xml/init/ptr.cpp +++ b/library/cpp/xml/init/ptr.cpp @@ -1 +1 @@ -#include "ptr.h" +#include "ptr.h" diff --git a/library/cpp/xml/init/ptr.h b/library/cpp/xml/init/ptr.h index 7387c7cc40..33e433a858 100644 --- a/library/cpp/xml/init/ptr.h +++ b/library/cpp/xml/init/ptr.h @@ -9,7 +9,7 @@ #include <libxml/uri.h> #include <libxml/xmlschemas.h> -template <class T, void (*DestroyFun)(T*)> +template <class T, void (*DestroyFun)(T*)> struct TFunctionDestroy { static inline void Destroy(T* t) noexcept { if (t) @@ -18,49 +18,49 @@ struct TFunctionDestroy { }; namespace NXml { -#define DEF_HOLDER(type, free) typedef THolder<type, TFunctionDestroy<type, free>> T##type##Holder -#define DEF_PTR(type, free) typedef TAutoPtr<type, TFunctionDestroy<type, free>> T##type##Ptr +#define DEF_HOLDER(type, free) typedef THolder<type, TFunctionDestroy<type, free>> T##type##Holder +#define DEF_PTR(type, free) typedef TAutoPtr<type, TFunctionDestroy<type, free>> T##type##Ptr - // define xmlDocPtr -> TxmlDocHolder TxmlDocPtr - DEF_HOLDER(xmlDoc, xmlFreeDoc); - DEF_PTR(xmlDoc, xmlFreeDoc); + // define xmlDocPtr -> TxmlDocHolder TxmlDocPtr + DEF_HOLDER(xmlDoc, xmlFreeDoc); + DEF_PTR(xmlDoc, xmlFreeDoc); - // xmlXPathContextPtr xpathCtx; - DEF_HOLDER(xmlXPathContext, xmlXPathFreeContext); - DEF_PTR(xmlXPathContext, xmlXPathFreeContext); + // xmlXPathContextPtr xpathCtx; + DEF_HOLDER(xmlXPathContext, xmlXPathFreeContext); + DEF_PTR(xmlXPathContext, xmlXPathFreeContext); - // xmlXPathObjectPtr xpathObj; - DEF_HOLDER(xmlXPathObject, xmlXPathFreeObject); - DEF_PTR(xmlXPathObject, xmlXPathFreeObject); + // xmlXPathObjectPtr xpathObj; + DEF_HOLDER(xmlXPathObject, xmlXPathFreeObject); + DEF_PTR(xmlXPathObject, xmlXPathFreeObject); - // xmlNodeSetPtr nodes - DEF_HOLDER(xmlNodeSet, xmlXPathFreeNodeSet); - DEF_PTR(xmlNodeSet, xmlXPathFreeNodeSet); + // xmlNodeSetPtr nodes + DEF_HOLDER(xmlNodeSet, xmlXPathFreeNodeSet); + DEF_PTR(xmlNodeSet, xmlXPathFreeNodeSet); - // xmlSchemaParserCtxtPtr ctxt; + // xmlSchemaParserCtxtPtr ctxt; DEF_HOLDER(xmlSchemaParserCtxt, xmlSchemaFreeParserCtxt); - DEF_PTR(xmlSchemaParserCtxt, xmlSchemaFreeParserCtxt); + DEF_PTR(xmlSchemaParserCtxt, xmlSchemaFreeParserCtxt); - // xmlSchemaPtr schema; + // xmlSchemaPtr schema; DEF_HOLDER(xmlSchema, xmlSchemaFree); - DEF_PTR(xmlSchema, xmlSchemaFree); + DEF_PTR(xmlSchema, xmlSchemaFree); - // xmlSchemaValidCtxt ctxt; + // xmlSchemaValidCtxt ctxt; DEF_HOLDER(xmlSchemaValidCtxt, xmlSchemaFreeValidCtxt); - DEF_PTR(xmlSchemaValidCtxt, xmlSchemaFreeValidCtxt); + DEF_PTR(xmlSchemaValidCtxt, xmlSchemaFreeValidCtxt); - // xmlSaveCtxtPtr - inline void xmlFreeSave(xmlSaveCtxt* c) { - // returns int - xmlSaveClose(c); - } - DEF_HOLDER(xmlSaveCtxt, xmlFreeSave); - DEF_PTR(xmlSaveCtxt, xmlFreeSave); + // xmlSaveCtxtPtr + inline void xmlFreeSave(xmlSaveCtxt* c) { + // returns int + xmlSaveClose(c); + } + DEF_HOLDER(xmlSaveCtxt, xmlFreeSave); + DEF_PTR(xmlSaveCtxt, xmlFreeSave); - DEF_PTR(xmlURI, xmlFreeURI); - DEF_PTR(xmlNode, xmlFreeNode); - DEF_PTR(xmlParserCtxt, xmlFreeParserCtxt); - DEF_PTR(xmlParserInputBuffer, xmlFreeParserInputBuffer); + DEF_PTR(xmlURI, xmlFreeURI); + DEF_PTR(xmlNode, xmlFreeNode); + DEF_PTR(xmlParserCtxt, xmlFreeParserCtxt); + DEF_PTR(xmlParserInputBuffer, xmlFreeParserInputBuffer); DEF_PTR(xmlDtd, xmlFreeDtd); DEF_PTR(xmlValidCtxt, xmlFreeValidCtxt); diff --git a/library/cpp/xml/init/ya.make b/library/cpp/xml/init/ya.make index 5f3883c2c9..64a39af9d5 100644 --- a/library/cpp/xml/init/ya.make +++ b/library/cpp/xml/init/ya.make @@ -1,15 +1,15 @@ -LIBRARY() - -OWNER(pg) - -PEERDIR( - contrib/libs/libxml +LIBRARY() + +OWNER(pg) + +PEERDIR( + contrib/libs/libxml library/cpp/charset -) - -SRCS( - ptr.cpp - init.cpp -) - -END() +) + +SRCS( + ptr.cpp + init.cpp +) + +END() diff --git a/library/cpp/xml/ya.make b/library/cpp/xml/ya.make index 5a685b70e6..961ad39327 100644 --- a/library/cpp/xml/ya.make +++ b/library/cpp/xml/ya.make @@ -1,8 +1,8 @@ -RECURSE( +RECURSE( doc document/ut - encode - encode/ut + encode + encode/ut init parslib parslib/ut @@ -10,4 +10,4 @@ RECURSE( sax/ut validator validator/ut -) +) diff --git a/library/cpp/yson/consumer.cpp b/library/cpp/yson/consumer.cpp index 40ae452978..6f9f975bb6 100644 --- a/library/cpp/yson/consumer.cpp +++ b/library/cpp/yson/consumer.cpp @@ -4,12 +4,12 @@ namespace NYson { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// void TYsonConsumerBase::OnRaw(TStringBuf str, NYT::NYson::EYsonType type) { ParseYsonStringBuffer(str, this, type); - } - - //////////////////////////////////////////////////////////////////////////////// + } + //////////////////////////////////////////////////////////////////////////////// + } // namespace NYson diff --git a/library/cpp/yson/consumer.h b/library/cpp/yson/consumer.h index d5a9d66335..52d160c6cc 100644 --- a/library/cpp/yson/consumer.h +++ b/library/cpp/yson/consumer.h @@ -6,8 +6,8 @@ #include <util/system/defaults.h> namespace NYson { - struct TYsonConsumerBase + struct TYsonConsumerBase : public virtual NYT::NYson::IYsonConsumer { void OnRaw(TStringBuf ysonNode, NYT::NYson::EYsonType type) override; - }; + }; } // namespace NYson diff --git a/library/cpp/yson/detail.h b/library/cpp/yson/detail.h index 27f5e8ffff..47f2d3fa3a 100644 --- a/library/cpp/yson/detail.h +++ b/library/cpp/yson/detail.h @@ -11,796 +11,796 @@ #include <util/stream/input.h> namespace NYson { - namespace NDetail { - //////////////////////////////////////////////////////////////////////////////// - - //! Indicates the beginning of a list. - const char BeginListSymbol = '['; - //! Indicates the end of a list. - const char EndListSymbol = ']'; - - //! Indicates the beginning of a map. - const char BeginMapSymbol = '{'; - //! Indicates the end of a map. - const char EndMapSymbol = '}'; - - //! Indicates the beginning of an attribute map. - const char BeginAttributesSymbol = '<'; - //! Indicates the end of an attribute map. - const char EndAttributesSymbol = '>'; - - //! Separates items in lists. - const char ListItemSeparatorSymbol = ';'; - //! Separates items in maps, attributes. - const char KeyedItemSeparatorSymbol = ';'; - //! Separates keys from values in maps. - const char KeyValueSeparatorSymbol = '='; - - //! Indicates an entity. - const char EntitySymbol = '#'; - - //! Indicates end of stream. - const char EndSymbol = '\0'; - - //! Marks the beginning of a binary string literal. - const char StringMarker = '\x01'; - //! Marks the beginning of a binary i64 literal. - const char Int64Marker = '\x02'; - //! Marks the beginning of a binary double literal. - const char DoubleMarker = '\x03'; - //! Marks true and false values of boolean. - const char FalseMarker = '\x04'; - const char TrueMarker = '\x05'; - //! Marks the beginning of a binary ui64 literal. - const char Uint64Marker = '\x06'; - - //////////////////////////////////////////////////////////////////////////////// - - template <bool EnableLinePositionInfo> - class TPositionInfo; - - template <> - class TPositionInfo<true> { - private: - int Offset; - int Line; - int Column; - - public: - TPositionInfo() - : Offset(0) - , Line(1) - , Column(1) - { - } - - void OnRangeConsumed(const char* begin, const char* end) { - Offset += end - begin; - for (auto current = begin; current != end; ++current) { - ++Column; - if (*current == '\n') { //TODO: memchr - ++Line; - Column = 1; - } - } - } - }; - - template <> - class TPositionInfo<false> { - private: - int Offset; - - public: - TPositionInfo() - : Offset(0) - { - } - - void OnRangeConsumed(const char* begin, const char* end) { - Offset += end - begin; - } - }; - - template <class TBlockStream, class TPositionBase> - class TCharStream - : public TBlockStream, - public TPositionBase { - public: - TCharStream(const TBlockStream& blockStream) - : TBlockStream(blockStream) - { - } - - bool IsEmpty() const { - return TBlockStream::Begin() == TBlockStream::End(); - } - - template <bool AllowFinish> - void Refresh() { - while (IsEmpty() && !TBlockStream::IsFinished()) { - TBlockStream::RefreshBlock(); - } - if (IsEmpty() && TBlockStream::IsFinished() && !AllowFinish) { - ythrow TYsonException() << "Premature end of yson stream"; - } - } - - void Refresh() { - return Refresh<false>(); - } - - template <bool AllowFinish> - char GetChar() { - Refresh<AllowFinish>(); - return !IsEmpty() ? *TBlockStream::Begin() : '\0'; - } - - char GetChar() { - return GetChar<false>(); - } - - void Advance(size_t bytes) { - TPositionBase::OnRangeConsumed(TBlockStream::Begin(), TBlockStream::Begin() + bytes); - TBlockStream::Advance(bytes); - } - - size_t Length() const { - return TBlockStream::End() - TBlockStream::Begin(); - } - }; - - template <class TBaseStream> - class TCodedStream - : public TBaseStream { - private: - static const int MaxVarintBytes = 10; - static const int MaxVarint32Bytes = 5; - - const ui8* BeginByte() const { - return reinterpret_cast<const ui8*>(TBaseStream::Begin()); - } - - const ui8* EndByte() const { - return reinterpret_cast<const ui8*>(TBaseStream::End()); - } - - // Following functions is an adaptation Protobuf code from coded_stream.cc - bool ReadVarint32FromArray(ui32* value) { - // Fast path: We have enough bytes left in the buffer to guarantee that - // this read won't cross the end, so we can skip the checks. - const ui8* ptr = BeginByte(); - ui32 b; - ui32 result; - - b = *(ptr++); - result = (b & 0x7F); - if (!(b & 0x80)) - goto done; - b = *(ptr++); - result |= (b & 0x7F) << 7; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - result |= (b & 0x7F) << 14; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - result |= (b & 0x7F) << 21; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - result |= b << 28; - if (!(b & 0x80)) - goto done; - - // If the input is larger than 32 bits, we still need to read it all - // and discard the high-order bits. - - for (int i = 0; i < MaxVarintBytes - MaxVarint32Bytes; i++) { - b = *(ptr++); - if (!(b & 0x80)) - goto done; - } - - // We have overrun the maximum size of a Varint (10 bytes). Assume - // the data is corrupt. - return false; - - done: - TBaseStream::Advance(ptr - BeginByte()); - *value = result; - return true; - } - - bool ReadVarint32Fallback(ui32* value) { - if (BeginByte() + MaxVarint32Bytes <= EndByte() || - // Optimization: If the Varint ends at exactly the end of the buffer, - // we can detect that and still use the fast path. - (BeginByte() < EndByte() && !(EndByte()[-1] & 0x80))) - { - return ReadVarint32FromArray(value); - } else { - // Really slow case: we will incur the cost of an extra function call here, - // but moving this out of line reduces the size of this function, which - // improves the common case. In micro benchmarks, this is worth about 10-15% - return ReadVarint32Slow(value); - } - } - - bool ReadVarint32Slow(ui32* value) { - ui64 result; - // Directly invoke ReadVarint64Fallback, since we already tried to optimize - // for one-byte Varints. - if (ReadVarint64Fallback(&result)) { - *value = static_cast<ui32>(result); - return true; - } else { - return false; - } - } - - bool ReadVarint64Slow(ui64* value) { - // Slow path: This read might cross the end of the buffer, so we - // need to check and refresh the buffer if and when it does. - - ui64 result = 0; - int count = 0; - ui32 b; - - do { - if (count == MaxVarintBytes) { - return false; - } - while (BeginByte() == EndByte()) { - TBaseStream::Refresh(); - } - b = *BeginByte(); - result |= static_cast<ui64>(b & 0x7F) << (7 * count); - TBaseStream::Advance(1); - ++count; - } while (b & 0x80); - - *value = result; - return true; - } - - bool ReadVarint64Fallback(ui64* value) { - if (BeginByte() + MaxVarintBytes <= EndByte() || - // Optimization: If the Varint ends at exactly the end of the buffer, - // we can detect that and still use the fast path. - (BeginByte() < EndByte() && !(EndByte()[-1] & 0x80))) - { - // Fast path: We have enough bytes left in the buffer to guarantee that - // this read won't cross the end, so we can skip the checks. - - const ui8* ptr = BeginByte(); - ui32 b; - - // Splitting into 32-bit pieces gives better performance on 32-bit - // processors. - ui32 part0 = 0, part1 = 0, part2 = 0; - - b = *(ptr++); - part0 = (b & 0x7F); - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part0 |= (b & 0x7F) << 7; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part0 |= (b & 0x7F) << 14; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part0 |= (b & 0x7F) << 21; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part1 = (b & 0x7F); - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part1 |= (b & 0x7F) << 7; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part1 |= (b & 0x7F) << 14; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part1 |= (b & 0x7F) << 21; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part2 = (b & 0x7F); - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part2 |= (b & 0x7F) << 7; - if (!(b & 0x80)) - goto done; - - // We have overrun the maximum size of a Varint (10 bytes). The data - // must be corrupt. - return false; - - done: - TBaseStream::Advance(ptr - BeginByte()); - *value = (static_cast<ui64>(part0)) | - (static_cast<ui64>(part1) << 28) | - (static_cast<ui64>(part2) << 56); - return true; - } else { - return ReadVarint64Slow(value); - } - } - - public: - TCodedStream(const TBaseStream& baseStream) - : TBaseStream(baseStream) - { - } - - bool ReadVarint64(ui64* value) { - if (BeginByte() < EndByte() && *BeginByte() < 0x80) { - *value = *BeginByte(); - TBaseStream::Advance(1); - return true; - } else { - return ReadVarint64Fallback(value); - } - } - - bool ReadVarint32(ui32* value) { - if (BeginByte() < EndByte() && *BeginByte() < 0x80) { - *value = *BeginByte(); - TBaseStream::Advance(1); - return true; - } else { - return ReadVarint32Fallback(value); - } - } - }; - - enum ENumericResult { - Int64 = 0, - Uint64 = 1, - Double = 2 - }; - - template <class TBlockStream, bool EnableLinePositionInfo> - class TLexerBase - : public TCodedStream<TCharStream<TBlockStream, TPositionInfo<EnableLinePositionInfo>>> { - private: - using TBaseStream = TCodedStream<TCharStream<TBlockStream, TPositionInfo<EnableLinePositionInfo>>>; - TVector<char> Buffer_; - TMaybe<ui64> MemoryLimit_; - - void CheckMemoryLimit() { - if (MemoryLimit_ && Buffer_.capacity() > *MemoryLimit_) { + namespace NDetail { + //////////////////////////////////////////////////////////////////////////////// + + //! Indicates the beginning of a list. + const char BeginListSymbol = '['; + //! Indicates the end of a list. + const char EndListSymbol = ']'; + + //! Indicates the beginning of a map. + const char BeginMapSymbol = '{'; + //! Indicates the end of a map. + const char EndMapSymbol = '}'; + + //! Indicates the beginning of an attribute map. + const char BeginAttributesSymbol = '<'; + //! Indicates the end of an attribute map. + const char EndAttributesSymbol = '>'; + + //! Separates items in lists. + const char ListItemSeparatorSymbol = ';'; + //! Separates items in maps, attributes. + const char KeyedItemSeparatorSymbol = ';'; + //! Separates keys from values in maps. + const char KeyValueSeparatorSymbol = '='; + + //! Indicates an entity. + const char EntitySymbol = '#'; + + //! Indicates end of stream. + const char EndSymbol = '\0'; + + //! Marks the beginning of a binary string literal. + const char StringMarker = '\x01'; + //! Marks the beginning of a binary i64 literal. + const char Int64Marker = '\x02'; + //! Marks the beginning of a binary double literal. + const char DoubleMarker = '\x03'; + //! Marks true and false values of boolean. + const char FalseMarker = '\x04'; + const char TrueMarker = '\x05'; + //! Marks the beginning of a binary ui64 literal. + const char Uint64Marker = '\x06'; + + //////////////////////////////////////////////////////////////////////////////// + + template <bool EnableLinePositionInfo> + class TPositionInfo; + + template <> + class TPositionInfo<true> { + private: + int Offset; + int Line; + int Column; + + public: + TPositionInfo() + : Offset(0) + , Line(1) + , Column(1) + { + } + + void OnRangeConsumed(const char* begin, const char* end) { + Offset += end - begin; + for (auto current = begin; current != end; ++current) { + ++Column; + if (*current == '\n') { //TODO: memchr + ++Line; + Column = 1; + } + } + } + }; + + template <> + class TPositionInfo<false> { + private: + int Offset; + + public: + TPositionInfo() + : Offset(0) + { + } + + void OnRangeConsumed(const char* begin, const char* end) { + Offset += end - begin; + } + }; + + template <class TBlockStream, class TPositionBase> + class TCharStream + : public TBlockStream, + public TPositionBase { + public: + TCharStream(const TBlockStream& blockStream) + : TBlockStream(blockStream) + { + } + + bool IsEmpty() const { + return TBlockStream::Begin() == TBlockStream::End(); + } + + template <bool AllowFinish> + void Refresh() { + while (IsEmpty() && !TBlockStream::IsFinished()) { + TBlockStream::RefreshBlock(); + } + if (IsEmpty() && TBlockStream::IsFinished() && !AllowFinish) { + ythrow TYsonException() << "Premature end of yson stream"; + } + } + + void Refresh() { + return Refresh<false>(); + } + + template <bool AllowFinish> + char GetChar() { + Refresh<AllowFinish>(); + return !IsEmpty() ? *TBlockStream::Begin() : '\0'; + } + + char GetChar() { + return GetChar<false>(); + } + + void Advance(size_t bytes) { + TPositionBase::OnRangeConsumed(TBlockStream::Begin(), TBlockStream::Begin() + bytes); + TBlockStream::Advance(bytes); + } + + size_t Length() const { + return TBlockStream::End() - TBlockStream::Begin(); + } + }; + + template <class TBaseStream> + class TCodedStream + : public TBaseStream { + private: + static const int MaxVarintBytes = 10; + static const int MaxVarint32Bytes = 5; + + const ui8* BeginByte() const { + return reinterpret_cast<const ui8*>(TBaseStream::Begin()); + } + + const ui8* EndByte() const { + return reinterpret_cast<const ui8*>(TBaseStream::End()); + } + + // Following functions is an adaptation Protobuf code from coded_stream.cc + bool ReadVarint32FromArray(ui32* value) { + // Fast path: We have enough bytes left in the buffer to guarantee that + // this read won't cross the end, so we can skip the checks. + const ui8* ptr = BeginByte(); + ui32 b; + ui32 result; + + b = *(ptr++); + result = (b & 0x7F); + if (!(b & 0x80)) + goto done; + b = *(ptr++); + result |= (b & 0x7F) << 7; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + result |= (b & 0x7F) << 14; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + result |= (b & 0x7F) << 21; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + result |= b << 28; + if (!(b & 0x80)) + goto done; + + // If the input is larger than 32 bits, we still need to read it all + // and discard the high-order bits. + + for (int i = 0; i < MaxVarintBytes - MaxVarint32Bytes; i++) { + b = *(ptr++); + if (!(b & 0x80)) + goto done; + } + + // We have overrun the maximum size of a Varint (10 bytes). Assume + // the data is corrupt. + return false; + + done: + TBaseStream::Advance(ptr - BeginByte()); + *value = result; + return true; + } + + bool ReadVarint32Fallback(ui32* value) { + if (BeginByte() + MaxVarint32Bytes <= EndByte() || + // Optimization: If the Varint ends at exactly the end of the buffer, + // we can detect that and still use the fast path. + (BeginByte() < EndByte() && !(EndByte()[-1] & 0x80))) + { + return ReadVarint32FromArray(value); + } else { + // Really slow case: we will incur the cost of an extra function call here, + // but moving this out of line reduces the size of this function, which + // improves the common case. In micro benchmarks, this is worth about 10-15% + return ReadVarint32Slow(value); + } + } + + bool ReadVarint32Slow(ui32* value) { + ui64 result; + // Directly invoke ReadVarint64Fallback, since we already tried to optimize + // for one-byte Varints. + if (ReadVarint64Fallback(&result)) { + *value = static_cast<ui32>(result); + return true; + } else { + return false; + } + } + + bool ReadVarint64Slow(ui64* value) { + // Slow path: This read might cross the end of the buffer, so we + // need to check and refresh the buffer if and when it does. + + ui64 result = 0; + int count = 0; + ui32 b; + + do { + if (count == MaxVarintBytes) { + return false; + } + while (BeginByte() == EndByte()) { + TBaseStream::Refresh(); + } + b = *BeginByte(); + result |= static_cast<ui64>(b & 0x7F) << (7 * count); + TBaseStream::Advance(1); + ++count; + } while (b & 0x80); + + *value = result; + return true; + } + + bool ReadVarint64Fallback(ui64* value) { + if (BeginByte() + MaxVarintBytes <= EndByte() || + // Optimization: If the Varint ends at exactly the end of the buffer, + // we can detect that and still use the fast path. + (BeginByte() < EndByte() && !(EndByte()[-1] & 0x80))) + { + // Fast path: We have enough bytes left in the buffer to guarantee that + // this read won't cross the end, so we can skip the checks. + + const ui8* ptr = BeginByte(); + ui32 b; + + // Splitting into 32-bit pieces gives better performance on 32-bit + // processors. + ui32 part0 = 0, part1 = 0, part2 = 0; + + b = *(ptr++); + part0 = (b & 0x7F); + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part0 |= (b & 0x7F) << 7; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part0 |= (b & 0x7F) << 14; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part0 |= (b & 0x7F) << 21; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part1 = (b & 0x7F); + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part1 |= (b & 0x7F) << 7; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part1 |= (b & 0x7F) << 14; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part1 |= (b & 0x7F) << 21; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part2 = (b & 0x7F); + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part2 |= (b & 0x7F) << 7; + if (!(b & 0x80)) + goto done; + + // We have overrun the maximum size of a Varint (10 bytes). The data + // must be corrupt. + return false; + + done: + TBaseStream::Advance(ptr - BeginByte()); + *value = (static_cast<ui64>(part0)) | + (static_cast<ui64>(part1) << 28) | + (static_cast<ui64>(part2) << 56); + return true; + } else { + return ReadVarint64Slow(value); + } + } + + public: + TCodedStream(const TBaseStream& baseStream) + : TBaseStream(baseStream) + { + } + + bool ReadVarint64(ui64* value) { + if (BeginByte() < EndByte() && *BeginByte() < 0x80) { + *value = *BeginByte(); + TBaseStream::Advance(1); + return true; + } else { + return ReadVarint64Fallback(value); + } + } + + bool ReadVarint32(ui32* value) { + if (BeginByte() < EndByte() && *BeginByte() < 0x80) { + *value = *BeginByte(); + TBaseStream::Advance(1); + return true; + } else { + return ReadVarint32Fallback(value); + } + } + }; + + enum ENumericResult { + Int64 = 0, + Uint64 = 1, + Double = 2 + }; + + template <class TBlockStream, bool EnableLinePositionInfo> + class TLexerBase + : public TCodedStream<TCharStream<TBlockStream, TPositionInfo<EnableLinePositionInfo>>> { + private: + using TBaseStream = TCodedStream<TCharStream<TBlockStream, TPositionInfo<EnableLinePositionInfo>>>; + TVector<char> Buffer_; + TMaybe<ui64> MemoryLimit_; + + void CheckMemoryLimit() { + if (MemoryLimit_ && Buffer_.capacity() > *MemoryLimit_) { ythrow TYsonException() << "Memory limit exceeded while parsing YSON stream: allocated " << Buffer_.capacity() << ", limit " << (*MemoryLimit_); - } - } - - public: - TLexerBase(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit) - : TBaseStream(blockStream) - , MemoryLimit_(memoryLimit) - { - } - - protected: - /// Lexer routines - - template <bool AllowFinish> - ENumericResult ReadNumeric(TStringBuf* value) { - Buffer_.clear(); - ENumericResult result = ENumericResult::Int64; - while (true) { - char ch = TBaseStream::template GetChar<AllowFinish>(); - if (isdigit(ch) || ch == '+' || ch == '-') { // Seems like it can't be '+' or '-' - Buffer_.push_back(ch); - } else if (ch == '.' || ch == 'e' || ch == 'E') { - Buffer_.push_back(ch); - result = ENumericResult::Double; - } else if (ch == 'u') { - Buffer_.push_back(ch); - result = ENumericResult::Uint64; - } else if (isalpha(ch)) { + } + } + + public: + TLexerBase(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit) + : TBaseStream(blockStream) + , MemoryLimit_(memoryLimit) + { + } + + protected: + /// Lexer routines + + template <bool AllowFinish> + ENumericResult ReadNumeric(TStringBuf* value) { + Buffer_.clear(); + ENumericResult result = ENumericResult::Int64; + while (true) { + char ch = TBaseStream::template GetChar<AllowFinish>(); + if (isdigit(ch) || ch == '+' || ch == '-') { // Seems like it can't be '+' or '-' + Buffer_.push_back(ch); + } else if (ch == '.' || ch == 'e' || ch == 'E') { + Buffer_.push_back(ch); + result = ENumericResult::Double; + } else if (ch == 'u') { + Buffer_.push_back(ch); + result = ENumericResult::Uint64; + } else if (isalpha(ch)) { ythrow TYsonException() << "Unexpected '" << ch << "' in numeric literal"; - } else { - break; - } - CheckMemoryLimit(); - TBaseStream::Advance(1); - } + } else { + break; + } + CheckMemoryLimit(); + TBaseStream::Advance(1); + } - *value = TStringBuf(Buffer_.data(), Buffer_.size()); - return result; + *value = TStringBuf(Buffer_.data(), Buffer_.size()); + return result; } - template <bool AllowFinish> - double ReadNanOrInf() { + template <bool AllowFinish> + double ReadNanOrInf() { static const TStringBuf nanString = "nan"; static const TStringBuf infString = "inf"; static const TStringBuf plusInfString = "+inf"; static const TStringBuf minusInfString = "-inf"; - TStringBuf expectedString; - double expectedValue; - char ch = TBaseStream::template GetChar<AllowFinish>(); - switch (ch) { - case '+': - expectedString = plusInfString; - expectedValue = std::numeric_limits<double>::infinity(); - break; - case '-': - expectedString = minusInfString; - expectedValue = -std::numeric_limits<double>::infinity(); - break; - case 'i': - expectedString = infString; - expectedValue = std::numeric_limits<double>::infinity(); - break; - case 'n': - expectedString = nanString; - expectedValue = std::numeric_limits<double>::quiet_NaN(); - break; - default: - ythrow TYsonException() << "Incorrect %-literal prefix: '" << ch << "'"; - } + TStringBuf expectedString; + double expectedValue; + char ch = TBaseStream::template GetChar<AllowFinish>(); + switch (ch) { + case '+': + expectedString = plusInfString; + expectedValue = std::numeric_limits<double>::infinity(); + break; + case '-': + expectedString = minusInfString; + expectedValue = -std::numeric_limits<double>::infinity(); + break; + case 'i': + expectedString = infString; + expectedValue = std::numeric_limits<double>::infinity(); + break; + case 'n': + expectedString = nanString; + expectedValue = std::numeric_limits<double>::quiet_NaN(); + break; + default: + ythrow TYsonException() << "Incorrect %-literal prefix: '" << ch << "'"; + } for (size_t i = 0; i < expectedString.size(); ++i) { - if (expectedString[i] != ch) { - ythrow TYsonException() - << "Incorrect %-literal prefix " - << "'" << expectedString.SubStr(0, i) << ch << "'," - << "expected " << expectedString; - } - TBaseStream::Advance(1); - ch = TBaseStream::template GetChar<AllowFinish>(); - } - - return expectedValue; - } - - void ReadQuotedString(TStringBuf* value) { - Buffer_.clear(); - while (true) { - if (TBaseStream::IsEmpty()) { - TBaseStream::Refresh(); - } - char ch = *TBaseStream::Begin(); - TBaseStream::Advance(1); - if (ch != '"') { - Buffer_.push_back(ch); - } else { - // We must count the number of '\' at the end of StringValue - // to check if it's not \" - int slashCount = 0; - int length = Buffer_.size(); - while (slashCount < length && Buffer_[length - 1 - slashCount] == '\\') { - ++slashCount; - } - if (slashCount % 2 == 0) { - break; - } else { - Buffer_.push_back(ch); - } - } - CheckMemoryLimit(); - } - - auto unquotedValue = UnescapeC(Buffer_.data(), Buffer_.size()); - Buffer_.clear(); - Buffer_.insert(Buffer_.end(), unquotedValue.data(), unquotedValue.data() + unquotedValue.size()); - CheckMemoryLimit(); - *value = TStringBuf(Buffer_.data(), Buffer_.size()); - } - - template <bool AllowFinish> - void ReadUnquotedString(TStringBuf* value) { - Buffer_.clear(); - while (true) { - char ch = TBaseStream::template GetChar<AllowFinish>(); - if (isalpha(ch) || isdigit(ch) || - ch == '_' || ch == '-' || ch == '%' || ch == '.') { - Buffer_.push_back(ch); - } else { - break; - } - CheckMemoryLimit(); - TBaseStream::Advance(1); + if (expectedString[i] != ch) { + ythrow TYsonException() + << "Incorrect %-literal prefix " + << "'" << expectedString.SubStr(0, i) << ch << "'," + << "expected " << expectedString; + } + TBaseStream::Advance(1); + ch = TBaseStream::template GetChar<AllowFinish>(); + } + + return expectedValue; + } + + void ReadQuotedString(TStringBuf* value) { + Buffer_.clear(); + while (true) { + if (TBaseStream::IsEmpty()) { + TBaseStream::Refresh(); + } + char ch = *TBaseStream::Begin(); + TBaseStream::Advance(1); + if (ch != '"') { + Buffer_.push_back(ch); + } else { + // We must count the number of '\' at the end of StringValue + // to check if it's not \" + int slashCount = 0; + int length = Buffer_.size(); + while (slashCount < length && Buffer_[length - 1 - slashCount] == '\\') { + ++slashCount; + } + if (slashCount % 2 == 0) { + break; + } else { + Buffer_.push_back(ch); + } + } + CheckMemoryLimit(); + } + + auto unquotedValue = UnescapeC(Buffer_.data(), Buffer_.size()); + Buffer_.clear(); + Buffer_.insert(Buffer_.end(), unquotedValue.data(), unquotedValue.data() + unquotedValue.size()); + CheckMemoryLimit(); + *value = TStringBuf(Buffer_.data(), Buffer_.size()); + } + + template <bool AllowFinish> + void ReadUnquotedString(TStringBuf* value) { + Buffer_.clear(); + while (true) { + char ch = TBaseStream::template GetChar<AllowFinish>(); + if (isalpha(ch) || isdigit(ch) || + ch == '_' || ch == '-' || ch == '%' || ch == '.') { + Buffer_.push_back(ch); + } else { + break; + } + CheckMemoryLimit(); + TBaseStream::Advance(1); } - *value = TStringBuf(Buffer_.data(), Buffer_.size()); + *value = TStringBuf(Buffer_.data(), Buffer_.size()); } - void ReadUnquotedString(TStringBuf* value) { - return ReadUnquotedString<false>(value); + void ReadUnquotedString(TStringBuf* value) { + return ReadUnquotedString<false>(value); } - void ReadBinaryString(TStringBuf* value) { - ui32 ulength = 0; - if (!TBaseStream::ReadVarint32(&ulength)) { - ythrow TYsonException() << "Error parsing varint value"; - } + void ReadBinaryString(TStringBuf* value) { + ui32 ulength = 0; + if (!TBaseStream::ReadVarint32(&ulength)) { + ythrow TYsonException() << "Error parsing varint value"; + } - i32 length = ZigZagDecode32(ulength); - if (length < 0) { + i32 length = ZigZagDecode32(ulength); + if (length < 0) { ythrow TYsonException() << "Negative binary string literal length " << length; - } - - if (TBaseStream::Begin() + length <= TBaseStream::End()) { - *value = TStringBuf(TBaseStream::Begin(), length); - TBaseStream::Advance(length); - } else { // reading in Buffer - size_t needToRead = length; - Buffer_.clear(); - while (needToRead) { - if (TBaseStream::IsEmpty()) { - TBaseStream::Refresh(); - continue; - } - size_t readingBytes = Min(needToRead, TBaseStream::Length()); - - Buffer_.insert(Buffer_.end(), TBaseStream::Begin(), TBaseStream::Begin() + readingBytes); - CheckMemoryLimit(); - needToRead -= readingBytes; - TBaseStream::Advance(readingBytes); - } - *value = TStringBuf(Buffer_.data(), Buffer_.size()); + } + + if (TBaseStream::Begin() + length <= TBaseStream::End()) { + *value = TStringBuf(TBaseStream::Begin(), length); + TBaseStream::Advance(length); + } else { // reading in Buffer + size_t needToRead = length; + Buffer_.clear(); + while (needToRead) { + if (TBaseStream::IsEmpty()) { + TBaseStream::Refresh(); + continue; + } + size_t readingBytes = Min(needToRead, TBaseStream::Length()); + + Buffer_.insert(Buffer_.end(), TBaseStream::Begin(), TBaseStream::Begin() + readingBytes); + CheckMemoryLimit(); + needToRead -= readingBytes; + TBaseStream::Advance(readingBytes); + } + *value = TStringBuf(Buffer_.data(), Buffer_.size()); } } - template <bool AllowFinish> - bool ReadBoolean() { - Buffer_.clear(); + template <bool AllowFinish> + bool ReadBoolean() { + Buffer_.clear(); static TStringBuf trueString = "true"; static TStringBuf falseString = "false"; - auto throwIncorrectBoolean = [&]() { + auto throwIncorrectBoolean = [&]() { ythrow TYsonException() << "Incorrect boolean string " << TString(Buffer_.data(), Buffer_.size()); - }; + }; Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); TBaseStream::Advance(1); - if (Buffer_[0] == trueString[0]) { + if (Buffer_[0] == trueString[0]) { for (size_t i = 1; i < trueString.size(); ++i) { - Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); - TBaseStream::Advance(1); - if (Buffer_.back() != trueString[i]) { - throwIncorrectBoolean(); - } - } - return true; - } else if (Buffer_[0] == falseString[0]) { + Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); + TBaseStream::Advance(1); + if (Buffer_.back() != trueString[i]) { + throwIncorrectBoolean(); + } + } + return true; + } else if (Buffer_[0] == falseString[0]) { for (size_t i = 1; i < falseString.size(); ++i) { - Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); - TBaseStream::Advance(1); - if (Buffer_.back() != falseString[i]) { - throwIncorrectBoolean(); - } - } - return false; - } else { + Buffer_.push_back(TBaseStream::template GetChar<AllowFinish>()); + TBaseStream::Advance(1); + if (Buffer_.back() != falseString[i]) { + throwIncorrectBoolean(); + } + } + return false; + } else { throwIncorrectBoolean(); } - - Y_FAIL("unreachable"); - ; - } - - void ReadBinaryInt64(i64* result) { - ui64 uvalue; - if (!TBaseStream::ReadVarint64(&uvalue)) { - ythrow TYsonException() << "Error parsing varint value"; - } - *result = ZigZagDecode64(uvalue); - } - - void ReadBinaryUint64(ui64* result) { - ui64 uvalue; - if (!TBaseStream::ReadVarint64(&uvalue)) { - ythrow TYsonException() << "Error parsing varint value"; + + Y_FAIL("unreachable"); + ; + } + + void ReadBinaryInt64(i64* result) { + ui64 uvalue; + if (!TBaseStream::ReadVarint64(&uvalue)) { + ythrow TYsonException() << "Error parsing varint value"; } - *result = uvalue; - } - - void ReadBinaryDouble(double* value) { - size_t needToRead = sizeof(double); - - while (needToRead != 0) { - if (TBaseStream::IsEmpty()) { - TBaseStream::Refresh(); - continue; - } - - size_t chunkSize = Min(needToRead, TBaseStream::Length()); - if (chunkSize == 0) { - ythrow TYsonException() << "Error parsing binary double literal"; - } - std::copy( - TBaseStream::Begin(), - TBaseStream::Begin() + chunkSize, - reinterpret_cast<char*>(value) + (sizeof(double) - needToRead)); - needToRead -= chunkSize; - TBaseStream::Advance(chunkSize); - } - } - - /// Helpers - void SkipCharToken(char symbol) { - char ch = SkipSpaceAndGetChar(); - if (ch != symbol) { + *result = ZigZagDecode64(uvalue); + } + + void ReadBinaryUint64(ui64* result) { + ui64 uvalue; + if (!TBaseStream::ReadVarint64(&uvalue)) { + ythrow TYsonException() << "Error parsing varint value"; + } + *result = uvalue; + } + + void ReadBinaryDouble(double* value) { + size_t needToRead = sizeof(double); + + while (needToRead != 0) { + if (TBaseStream::IsEmpty()) { + TBaseStream::Refresh(); + continue; + } + + size_t chunkSize = Min(needToRead, TBaseStream::Length()); + if (chunkSize == 0) { + ythrow TYsonException() << "Error parsing binary double literal"; + } + std::copy( + TBaseStream::Begin(), + TBaseStream::Begin() + chunkSize, + reinterpret_cast<char*>(value) + (sizeof(double) - needToRead)); + needToRead -= chunkSize; + TBaseStream::Advance(chunkSize); + } + } + + /// Helpers + void SkipCharToken(char symbol) { + char ch = SkipSpaceAndGetChar(); + if (ch != symbol) { ythrow TYsonException() << "Expected '" << symbol << "' but found '" << ch << "'"; + } + + TBaseStream::Advance(1); + } + + static bool IsSpaceFast(char ch) { + static const ui8 lookupTable[] = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + return lookupTable[static_cast<ui8>(ch)]; + } + + template <bool AllowFinish> + char SkipSpaceAndGetChar() { + if (!TBaseStream::IsEmpty()) { + char ch = *TBaseStream::Begin(); + if (!IsSpaceFast(ch)) { + return ch; + } + } + return SkipSpaceAndGetCharFallback<AllowFinish>(); + } + + char SkipSpaceAndGetChar() { + return SkipSpaceAndGetChar<false>(); + } + + template <bool AllowFinish> + char SkipSpaceAndGetCharFallback() { + while (true) { + if (TBaseStream::IsEmpty()) { + if (TBaseStream::IsFinished()) { + return '\0'; + } + TBaseStream::template Refresh<AllowFinish>(); + continue; + } + if (!IsSpaceFast(*TBaseStream::Begin())) { + break; + } + TBaseStream::Advance(1); } - - TBaseStream::Advance(1); - } - - static bool IsSpaceFast(char ch) { - static const ui8 lookupTable[] = - { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - return lookupTable[static_cast<ui8>(ch)]; - } - - template <bool AllowFinish> - char SkipSpaceAndGetChar() { - if (!TBaseStream::IsEmpty()) { - char ch = *TBaseStream::Begin(); - if (!IsSpaceFast(ch)) { - return ch; - } - } - return SkipSpaceAndGetCharFallback<AllowFinish>(); - } - - char SkipSpaceAndGetChar() { - return SkipSpaceAndGetChar<false>(); - } - - template <bool AllowFinish> - char SkipSpaceAndGetCharFallback() { - while (true) { - if (TBaseStream::IsEmpty()) { - if (TBaseStream::IsFinished()) { - return '\0'; - } - TBaseStream::template Refresh<AllowFinish>(); - continue; - } - if (!IsSpaceFast(*TBaseStream::Begin())) { - break; - } - TBaseStream::Advance(1); - } - return TBaseStream::template GetChar<AllowFinish>(); - } - }; - - //////////////////////////////////////////////////////////////////////////////// - - } - - //////////////////////////////////////////////////////////////////////////////// - - class TStringReader { - private: - const char* BeginPtr; - const char* EndPtr; - - public: - TStringReader() - : BeginPtr(nullptr) - , EndPtr(nullptr) - { - } - - TStringReader(const char* begin, const char* end) - : BeginPtr(begin) - , EndPtr(end) - { - } - - const char* Begin() const { - return BeginPtr; - } - - const char* End() const { - return EndPtr; - } - - void RefreshBlock() { - Y_FAIL("unreachable"); - } - - void Advance(size_t bytes) { - BeginPtr += bytes; - } - - bool IsFinished() const { - return true; - } - - void SetBuffer(const char* begin, const char* end) { - BeginPtr = begin; - EndPtr = end; - } - }; - - //////////////////////////////////////////////////////////////////////////////// - - class TStreamReader { - public: - TStreamReader( - IInputStream* stream, - char* buffer, - size_t bufferSize) - : Stream(stream) - , Buffer(buffer) - , BufferSize(bufferSize) - { - BeginPtr = EndPtr = Buffer; - FinishFlag = false; - } - - const char* Begin() const { - return BeginPtr; - } - - const char* End() const { - return EndPtr; - } - - void RefreshBlock() { - size_t bytes = Stream->Read(Buffer, BufferSize); - BeginPtr = Buffer; - EndPtr = Buffer + bytes; - FinishFlag = (bytes == 0); - } - - void Advance(size_t bytes) { - BeginPtr += bytes; - } - - bool IsFinished() const { - return FinishFlag; - } - - private: - IInputStream* Stream; - char* Buffer; - size_t BufferSize; - - const char* BeginPtr; - const char* EndPtr; - bool FinishFlag; - }; - - //////////////////////////////////////////////////////////////////////////////// + return TBaseStream::template GetChar<AllowFinish>(); + } + }; + + //////////////////////////////////////////////////////////////////////////////// + + } + + //////////////////////////////////////////////////////////////////////////////// + + class TStringReader { + private: + const char* BeginPtr; + const char* EndPtr; + + public: + TStringReader() + : BeginPtr(nullptr) + , EndPtr(nullptr) + { + } + + TStringReader(const char* begin, const char* end) + : BeginPtr(begin) + , EndPtr(end) + { + } + + const char* Begin() const { + return BeginPtr; + } + + const char* End() const { + return EndPtr; + } + + void RefreshBlock() { + Y_FAIL("unreachable"); + } + + void Advance(size_t bytes) { + BeginPtr += bytes; + } + + bool IsFinished() const { + return true; + } + + void SetBuffer(const char* begin, const char* end) { + BeginPtr = begin; + EndPtr = end; + } + }; + + //////////////////////////////////////////////////////////////////////////////// + + class TStreamReader { + public: + TStreamReader( + IInputStream* stream, + char* buffer, + size_t bufferSize) + : Stream(stream) + , Buffer(buffer) + , BufferSize(bufferSize) + { + BeginPtr = EndPtr = Buffer; + FinishFlag = false; + } + + const char* Begin() const { + return BeginPtr; + } + + const char* End() const { + return EndPtr; + } + + void RefreshBlock() { + size_t bytes = Stream->Read(Buffer, BufferSize); + BeginPtr = Buffer; + EndPtr = Buffer + bytes; + FinishFlag = (bytes == 0); + } + + void Advance(size_t bytes) { + BeginPtr += bytes; + } + + bool IsFinished() const { + return FinishFlag; + } + + private: + IInputStream* Stream; + char* Buffer; + size_t BufferSize; + + const char* BeginPtr; + const char* EndPtr; + bool FinishFlag; + }; + + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/format.h b/library/cpp/yson/format.h index 2ff6dc9f6e..77abfba24f 100644 --- a/library/cpp/yson/format.h +++ b/library/cpp/yson/format.h @@ -3,23 +3,23 @@ #include "token.h" namespace NYson { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - const ETokenType BeginListToken = LeftBracket; - const ETokenType EndListToken = RightBracket; + const ETokenType BeginListToken = LeftBracket; + const ETokenType EndListToken = RightBracket; - const ETokenType BeginMapToken = LeftBrace; - const ETokenType EndMapToken = RightBrace; + const ETokenType BeginMapToken = LeftBrace; + const ETokenType EndMapToken = RightBrace; - const ETokenType BeginAttributesToken = LeftAngle; - const ETokenType EndAttributesToken = RightAngle; + const ETokenType BeginAttributesToken = LeftAngle; + const ETokenType EndAttributesToken = RightAngle; - const ETokenType ListItemSeparatorToken = Semicolon; - const ETokenType KeyedItemSeparatorToken = Semicolon; - const ETokenType KeyValueSeparatorToken = Equals; + const ETokenType ListItemSeparatorToken = Semicolon; + const ETokenType KeyedItemSeparatorToken = Semicolon; + const ETokenType KeyValueSeparatorToken = Equals; - const ETokenType EntityToken = Hash; + const ETokenType EntityToken = Hash; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/json/json_writer.cpp b/library/cpp/yson/json/json_writer.cpp index 87481256ec..f65144bd67 100644 --- a/library/cpp/yson/json/json_writer.cpp +++ b/library/cpp/yson/json/json_writer.cpp @@ -3,20 +3,20 @@ #include <library/cpp/json/json_writer.h> namespace NYT { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - static bool IsSpecialJsonKey(const TStringBuf& key) { - return key.size() > 0 && key[0] == '$'; - } + static bool IsSpecialJsonKey(const TStringBuf& key) { + return key.size() > 0 && key[0] == '$'; + } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - TJsonWriter::TJsonWriter( - IOutputStream* output, + TJsonWriter::TJsonWriter( + IOutputStream* output, ::NYson::EYsonType type, - EJsonFormat format, - EJsonAttributesMode attributesMode, - ESerializedBoolFormat booleanFormat) + EJsonFormat format, + EJsonAttributesMode attributesMode, + ESerializedBoolFormat booleanFormat) : TJsonWriter( output, NJson::TJsonWriterConfig{}.SetFormatOutput(format == JF_PRETTY), @@ -32,189 +32,189 @@ namespace NYT { ::NYson::EYsonType type, EJsonAttributesMode attributesMode, ESerializedBoolFormat booleanFormat) - : Output(output) - , Type(type) - , AttributesMode(attributesMode) - , BooleanFormat(booleanFormat) - , Depth(0) - { + : Output(output) + , Type(type) + , AttributesMode(attributesMode) + , BooleanFormat(booleanFormat) + , Depth(0) + { if (Type == ::NYson::EYsonType::MapFragment) { ythrow ::NYson::TYsonException() << ("Map fragments are not supported by Json"); - } + } - UnderlyingJsonWriter.Reset(new NJson::TJsonWriter( - output, + UnderlyingJsonWriter.Reset(new NJson::TJsonWriter( + output, config)); - JsonWriter = UnderlyingJsonWriter.Get(); - HasAttributes = false; - InAttributesBalance = 0; - } - - void TJsonWriter::EnterNode() { - if (AttributesMode == JAM_NEVER) { - HasAttributes = false; - } else if (AttributesMode == JAM_ON_DEMAND) { - // Do nothing - } else if (AttributesMode == JAM_ALWAYS) { - if (!HasAttributes) { - JsonWriter->OpenMap(); - JsonWriter->Write("$attributes"); - JsonWriter->OpenMap(); - JsonWriter->CloseMap(); - } - HasAttributes = true; - } - HasUnfoldedStructureStack.push_back(HasAttributes); - - if (HasAttributes) { - JsonWriter->Write("$value"); - HasAttributes = false; - } - - Depth += 1; - } - - void TJsonWriter::LeaveNode() { - Y_ASSERT(!HasUnfoldedStructureStack.empty()); - if (HasUnfoldedStructureStack.back()) { - // Close map of the {$attributes, $value} - JsonWriter->CloseMap(); - } - HasUnfoldedStructureStack.pop_back(); - - Depth -= 1; - + JsonWriter = UnderlyingJsonWriter.Get(); + HasAttributes = false; + InAttributesBalance = 0; + } + + void TJsonWriter::EnterNode() { + if (AttributesMode == JAM_NEVER) { + HasAttributes = false; + } else if (AttributesMode == JAM_ON_DEMAND) { + // Do nothing + } else if (AttributesMode == JAM_ALWAYS) { + if (!HasAttributes) { + JsonWriter->OpenMap(); + JsonWriter->Write("$attributes"); + JsonWriter->OpenMap(); + JsonWriter->CloseMap(); + } + HasAttributes = true; + } + HasUnfoldedStructureStack.push_back(HasAttributes); + + if (HasAttributes) { + JsonWriter->Write("$value"); + HasAttributes = false; + } + + Depth += 1; + } + + void TJsonWriter::LeaveNode() { + Y_ASSERT(!HasUnfoldedStructureStack.empty()); + if (HasUnfoldedStructureStack.back()) { + // Close map of the {$attributes, $value} + JsonWriter->CloseMap(); + } + HasUnfoldedStructureStack.pop_back(); + + Depth -= 1; + if (Depth == 0 && Type == ::NYson::EYsonType::ListFragment && InAttributesBalance == 0) { - JsonWriter->Flush(); - Output->Write("\n"); - } + JsonWriter->Flush(); + Output->Write("\n"); + } } - bool TJsonWriter::IsWriteAllowed() { - if (AttributesMode == JAM_NEVER) { - return InAttributesBalance == 0; - } - return true; + bool TJsonWriter::IsWriteAllowed() { + if (AttributesMode == JAM_NEVER) { + return InAttributesBalance == 0; + } + return true; } void TJsonWriter::OnStringScalar(TStringBuf value) { - if (IsWriteAllowed()) { - EnterNode(); - WriteStringScalar(value); - LeaveNode(); - } - } - - void TJsonWriter::OnInt64Scalar(i64 value) { - if (IsWriteAllowed()) { - EnterNode(); - JsonWriter->Write(value); - LeaveNode(); - } - } - - void TJsonWriter::OnUint64Scalar(ui64 value) { - if (IsWriteAllowed()) { + if (IsWriteAllowed()) { + EnterNode(); + WriteStringScalar(value); + LeaveNode(); + } + } + + void TJsonWriter::OnInt64Scalar(i64 value) { + if (IsWriteAllowed()) { + EnterNode(); + JsonWriter->Write(value); + LeaveNode(); + } + } + + void TJsonWriter::OnUint64Scalar(ui64 value) { + if (IsWriteAllowed()) { + EnterNode(); + JsonWriter->Write(value); + LeaveNode(); + } + } + + void TJsonWriter::OnDoubleScalar(double value) { + if (IsWriteAllowed()) { + EnterNode(); + JsonWriter->Write(value); + LeaveNode(); + } + } + + void TJsonWriter::OnBooleanScalar(bool value) { + if (IsWriteAllowed()) { + if (BooleanFormat == SBF_STRING) { + OnStringScalar(value ? "true" : "false"); + } else { + EnterNode(); + JsonWriter->Write(value); + LeaveNode(); + } + } + } + + void TJsonWriter::OnEntity() { + if (IsWriteAllowed()) { EnterNode(); - JsonWriter->Write(value); + JsonWriter->WriteNull(); LeaveNode(); } } - void TJsonWriter::OnDoubleScalar(double value) { - if (IsWriteAllowed()) { - EnterNode(); - JsonWriter->Write(value); - LeaveNode(); - } - } - - void TJsonWriter::OnBooleanScalar(bool value) { - if (IsWriteAllowed()) { - if (BooleanFormat == SBF_STRING) { - OnStringScalar(value ? "true" : "false"); - } else { - EnterNode(); - JsonWriter->Write(value); - LeaveNode(); - } - } + void TJsonWriter::OnBeginList() { + if (IsWriteAllowed()) { + EnterNode(); + JsonWriter->OpenArray(); + } } - void TJsonWriter::OnEntity() { - if (IsWriteAllowed()) { - EnterNode(); - JsonWriter->WriteNull(); - LeaveNode(); - } + void TJsonWriter::OnListItem() { } - void TJsonWriter::OnBeginList() { - if (IsWriteAllowed()) { - EnterNode(); - JsonWriter->OpenArray(); - } - } + void TJsonWriter::OnEndList() { + if (IsWriteAllowed()) { + JsonWriter->CloseArray(); + LeaveNode(); + } + } - void TJsonWriter::OnListItem() { - } - - void TJsonWriter::OnEndList() { - if (IsWriteAllowed()) { - JsonWriter->CloseArray(); - LeaveNode(); - } - } - - void TJsonWriter::OnBeginMap() { - if (IsWriteAllowed()) { - EnterNode(); - JsonWriter->OpenMap(); - } + void TJsonWriter::OnBeginMap() { + if (IsWriteAllowed()) { + EnterNode(); + JsonWriter->OpenMap(); + } } void TJsonWriter::OnKeyedItem(TStringBuf name) { - if (IsWriteAllowed()) { - if (IsSpecialJsonKey(name)) { - WriteStringScalar(TString("$") + name); - } else { - WriteStringScalar(name); - } - } - } - - void TJsonWriter::OnEndMap() { - if (IsWriteAllowed()) { - JsonWriter->CloseMap(); - LeaveNode(); - } + if (IsWriteAllowed()) { + if (IsSpecialJsonKey(name)) { + WriteStringScalar(TString("$") + name); + } else { + WriteStringScalar(name); + } + } } - void TJsonWriter::OnBeginAttributes() { - InAttributesBalance += 1; - if (AttributesMode != JAM_NEVER) { - JsonWriter->OpenMap(); - JsonWriter->Write("$attributes"); - JsonWriter->OpenMap(); + void TJsonWriter::OnEndMap() { + if (IsWriteAllowed()) { + JsonWriter->CloseMap(); + LeaveNode(); } } - void TJsonWriter::OnEndAttributes() { - InAttributesBalance -= 1; - if (AttributesMode != JAM_NEVER) { - HasAttributes = true; - JsonWriter->CloseMap(); - } + void TJsonWriter::OnBeginAttributes() { + InAttributesBalance += 1; + if (AttributesMode != JAM_NEVER) { + JsonWriter->OpenMap(); + JsonWriter->Write("$attributes"); + JsonWriter->OpenMap(); + } } - void TJsonWriter::WriteStringScalar(const TStringBuf& value) { - JsonWriter->Write(value); + void TJsonWriter::OnEndAttributes() { + InAttributesBalance -= 1; + if (AttributesMode != JAM_NEVER) { + HasAttributes = true; + JsonWriter->CloseMap(); + } } - void TJsonWriter::Flush() { - JsonWriter->Flush(); + void TJsonWriter::WriteStringScalar(const TStringBuf& value) { + JsonWriter->Write(value); } - //////////////////////////////////////////////////////////////////////////////// + void TJsonWriter::Flush() { + JsonWriter->Flush(); + } + //////////////////////////////////////////////////////////////////////////////// + } diff --git a/library/cpp/yson/json/json_writer.h b/library/cpp/yson/json/json_writer.h index d84ac0de53..8dca12fe0e 100644 --- a/library/cpp/yson/json/json_writer.h +++ b/library/cpp/yson/json/json_writer.h @@ -8,33 +8,33 @@ #include <util/generic/vector.h> namespace NYT { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - enum EJsonFormat { - JF_TEXT, - JF_PRETTY - }; + enum EJsonFormat { + JF_TEXT, + JF_PRETTY + }; - enum EJsonAttributesMode { - JAM_NEVER, - JAM_ON_DEMAND, - JAM_ALWAYS - }; + enum EJsonAttributesMode { + JAM_NEVER, + JAM_ON_DEMAND, + JAM_ALWAYS + }; - enum ESerializedBoolFormat { - SBF_BOOLEAN, - SBF_STRING - }; + enum ESerializedBoolFormat { + SBF_BOOLEAN, + SBF_STRING + }; - class TJsonWriter + class TJsonWriter : public ::NYson::TYsonConsumerBase { - public: - TJsonWriter( - IOutputStream* output, + public: + TJsonWriter( + IOutputStream* output, ::NYson::EYsonType type = ::NYson::EYsonType::Node, - EJsonFormat format = JF_TEXT, - EJsonAttributesMode attributesMode = JAM_ON_DEMAND, - ESerializedBoolFormat booleanFormat = SBF_STRING); + EJsonFormat format = JF_TEXT, + EJsonAttributesMode attributesMode = JAM_ON_DEMAND, + ESerializedBoolFormat booleanFormat = SBF_STRING); TJsonWriter( IOutputStream* output, @@ -43,47 +43,47 @@ namespace NYT { EJsonAttributesMode attributesMode = JAM_ON_DEMAND, ESerializedBoolFormat booleanFormat = SBF_STRING); - void Flush(); + void Flush(); void OnStringScalar(TStringBuf value) override; - void OnInt64Scalar(i64 value) override; - void OnUint64Scalar(ui64 value) override; - void OnDoubleScalar(double value) override; - void OnBooleanScalar(bool value) override; + void OnInt64Scalar(i64 value) override; + void OnUint64Scalar(ui64 value) override; + void OnDoubleScalar(double value) override; + void OnBooleanScalar(bool value) override; - void OnEntity() override; + void OnEntity() override; - void OnBeginList() override; - void OnListItem() override; - void OnEndList() override; + void OnBeginList() override; + void OnListItem() override; + void OnEndList() override; - void OnBeginMap() override; + void OnBeginMap() override; void OnKeyedItem(TStringBuf key) override; - void OnEndMap() override; + void OnEndMap() override; - void OnBeginAttributes() override; - void OnEndAttributes() override; + void OnBeginAttributes() override; + void OnEndAttributes() override; - private: - THolder<NJson::TJsonWriter> UnderlyingJsonWriter; - NJson::TJsonWriter* JsonWriter; - IOutputStream* Output; + private: + THolder<NJson::TJsonWriter> UnderlyingJsonWriter; + NJson::TJsonWriter* JsonWriter; + IOutputStream* Output; ::NYson::EYsonType Type; - EJsonAttributesMode AttributesMode; - ESerializedBoolFormat BooleanFormat; + EJsonAttributesMode AttributesMode; + ESerializedBoolFormat BooleanFormat; - void WriteStringScalar(const TStringBuf& value); + void WriteStringScalar(const TStringBuf& value); - void EnterNode(); - void LeaveNode(); - bool IsWriteAllowed(); + void EnterNode(); + void LeaveNode(); + bool IsWriteAllowed(); - TVector<bool> HasUnfoldedStructureStack; - int InAttributesBalance; - bool HasAttributes; - int Depth; - }; + TVector<bool> HasUnfoldedStructureStack; + int InAttributesBalance; + bool HasAttributes; + int Depth; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// -} +} diff --git a/library/cpp/yson/json/yson2json_adapter.h b/library/cpp/yson/json/yson2json_adapter.h index da1bf5ba70..2a069c1f08 100644 --- a/library/cpp/yson/json/yson2json_adapter.h +++ b/library/cpp/yson/json/yson2json_adapter.h @@ -50,4 +50,4 @@ namespace NYT { ::NYson::TYsonConsumerBase* Impl_; TState State_; }; -} +} diff --git a/library/cpp/yson/lexer.cpp b/library/cpp/yson/lexer.cpp index 5eae94273b..da0021cc4f 100644 --- a/library/cpp/yson/lexer.cpp +++ b/library/cpp/yson/lexer.cpp @@ -5,39 +5,39 @@ #include <util/generic/ptr.h> namespace NYson { - //////////////////////////////////////////////////////////////////////////////// - - class TStatelessLexer::TImpl { - private: - THolder<TStatelessYsonLexerImplBase> Impl; - - public: - TImpl(bool enableLinePositionInfo = false) - : Impl(enableLinePositionInfo - ? static_cast<TStatelessYsonLexerImplBase*>(new TStatelesYsonLexerImpl<true>()) - : static_cast<TStatelessYsonLexerImplBase*>(new TStatelesYsonLexerImpl<false>())) - { - } - - size_t GetToken(const TStringBuf& data, TToken* token) { - return Impl->GetToken(data, token); - } - }; - - //////////////////////////////////////////////////////////////////////////////// - - TStatelessLexer::TStatelessLexer() - : Impl(new TImpl()) + //////////////////////////////////////////////////////////////////////////////// + + class TStatelessLexer::TImpl { + private: + THolder<TStatelessYsonLexerImplBase> Impl; + + public: + TImpl(bool enableLinePositionInfo = false) + : Impl(enableLinePositionInfo + ? static_cast<TStatelessYsonLexerImplBase*>(new TStatelesYsonLexerImpl<true>()) + : static_cast<TStatelessYsonLexerImplBase*>(new TStatelesYsonLexerImpl<false>())) + { + } + + size_t GetToken(const TStringBuf& data, TToken* token) { + return Impl->GetToken(data, token); + } + }; + + //////////////////////////////////////////////////////////////////////////////// + + TStatelessLexer::TStatelessLexer() + : Impl(new TImpl()) { } - TStatelessLexer::~TStatelessLexer() { - } + TStatelessLexer::~TStatelessLexer() { + } - size_t TStatelessLexer::GetToken(const TStringBuf& data, TToken* token) { - return Impl->GetToken(data, token); - } + size_t TStatelessLexer::GetToken(const TStringBuf& data, TToken* token) { + return Impl->GetToken(data, token); + } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/lexer.h b/library/cpp/yson/lexer.h index d9d701874d..3855192ba7 100644 --- a/library/cpp/yson/lexer.h +++ b/library/cpp/yson/lexer.h @@ -6,21 +6,21 @@ #include <util/generic/ptr.h> namespace NYson { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - class TStatelessLexer { - public: - TStatelessLexer(); + class TStatelessLexer { + public: + TStatelessLexer(); - ~TStatelessLexer(); + ~TStatelessLexer(); - size_t GetToken(const TStringBuf& data, TToken* token); + size_t GetToken(const TStringBuf& data, TToken* token); - private: - class TImpl; - THolder<TImpl> Impl; - }; + private: + class TImpl; + THolder<TImpl> Impl; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/lexer_detail.h b/library/cpp/yson/lexer_detail.h index 0bba30acdd..be61289cb1 100644 --- a/library/cpp/yson/lexer_detail.h +++ b/library/cpp/yson/lexer_detail.h @@ -4,58 +4,58 @@ #include "token.h" namespace NYson { - //////////////////////////////////////////////////////////////////////////////// - - namespace NDetail { - /*! \internal */ - //////////////////////////////////////////////////////////////////////////////// - - // EReadStartCase tree representation: - // Root = xb - // BinaryStringOrOtherSpecialToken = x0b - // BinaryString = 00b - // OtherSpecialToken = 10b - // Other = x1b - // BinaryScalar = xx01b - // BinaryInt64 = 0001b - // BinaryDouble = 0101b - // BinaryFalse = 1001b - // BinaryTrue = 1101b - // Other = xxx11b - // Quote = 00011b - // DigitOrMinus = 00111b - // String = 01011b - // Space = 01111b - // Plus = 10011b - // None = 10111b - // Percent = 11011b + //////////////////////////////////////////////////////////////////////////////// + + namespace NDetail { + /*! \internal */ + //////////////////////////////////////////////////////////////////////////////// + + // EReadStartCase tree representation: + // Root = xb + // BinaryStringOrOtherSpecialToken = x0b + // BinaryString = 00b + // OtherSpecialToken = 10b + // Other = x1b + // BinaryScalar = xx01b + // BinaryInt64 = 0001b + // BinaryDouble = 0101b + // BinaryFalse = 1001b + // BinaryTrue = 1101b + // Other = xxx11b + // Quote = 00011b + // DigitOrMinus = 00111b + // String = 01011b + // Space = 01111b + // Plus = 10011b + // None = 10111b + // Percent = 11011b enum EReadStartCase : unsigned { - BinaryString = 0, // = 00b - OtherSpecialToken = 2, // = 10b - - BinaryInt64 = 1, // = 001b - BinaryDouble = 5, // = 101b - BinaryFalse = 9, // = 1001b - BinaryTrue = 13, // = 1101b - BinaryUint64 = 17, // = 10001b - - Quote = 3, // = 00011b - DigitOrMinus = 7, // = 00111b - String = 11, // = 01011b - Space = 15, // = 01111b - Plus = 19, // = 10011b - None = 23, // = 10111b - Percent = 27 // = 11011b - }; - - template <class TBlockStream, bool EnableLinePositionInfo> - class TLexer - : public TLexerBase<TBlockStream, EnableLinePositionInfo> { - private: - using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>; - - static EReadStartCase GetStartState(char ch) { + BinaryString = 0, // = 00b + OtherSpecialToken = 2, // = 10b + + BinaryInt64 = 1, // = 001b + BinaryDouble = 5, // = 101b + BinaryFalse = 9, // = 1001b + BinaryTrue = 13, // = 1101b + BinaryUint64 = 17, // = 10001b + + Quote = 3, // = 00011b + DigitOrMinus = 7, // = 00111b + String = 11, // = 01011b + Space = 15, // = 01111b + Plus = 19, // = 10011b + None = 23, // = 10111b + Percent = 27 // = 11011b + }; + + template <class TBlockStream, bool EnableLinePositionInfo> + class TLexer + : public TLexerBase<TBlockStream, EnableLinePositionInfo> { + private: + using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>; + + static EReadStartCase GetStartState(char ch) { #define NN EReadStartCase::None #define BS EReadStartCase::BinaryString #define BI EReadStartCase::BinaryInt64 @@ -71,68 +71,68 @@ namespace NYson { #define PC EReadStartCase::Percent #define TT(name) (EReadStartCase(static_cast<ui8>(ETokenType::name) << 2) | EReadStartCase::OtherSpecialToken) - static const ui8 lookupTable[] = - { - NN, BS, BI, BD, BF, BT, BU, NN, NN, SP, SP, SP, SP, SP, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - - // 32 - SP, // ' ' - NN, // '!' - QU, // '"' - TT(Hash), // '#' - NN, // '$' - PC, // '%' - NN, // '&' - NN, // "'" - TT(LeftParenthesis), // '(' - TT(RightParenthesis), // ')' - NN, // '*' - PL, // '+' - TT(Comma), // ',' - DM, // '-' - NN, // '.' - NN, // '/' - - // 48 - DM, DM, DM, DM, DM, DM, DM, DM, DM, DM, // '0' - '9' - TT(Colon), // ':' - TT(Semicolon), // ';' - TT(LeftAngle), // '<' - TT(Equals), // '=' - TT(RightAngle), // '>' - NN, // '?' - - // 64 - NN, // '@' - ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'A' - 'M' - ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'N' - 'Z' - TT(LeftBracket), // '[' - NN, // '\' - TT(RightBracket), // ']' - NN, // '^' - ST, // '_' - - // 96 - NN, // '`' - - ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'a' - 'm' - ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'n' - 'z' - TT(LeftBrace), // '{' - NN, // '|' - TT(RightBrace), // '}' - NN, // '~' - NN, // '^?' non-printable - // 128 - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN}; + static const ui8 lookupTable[] = + { + NN, BS, BI, BD, BF, BT, BU, NN, NN, SP, SP, SP, SP, SP, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + + // 32 + SP, // ' ' + NN, // '!' + QU, // '"' + TT(Hash), // '#' + NN, // '$' + PC, // '%' + NN, // '&' + NN, // "'" + TT(LeftParenthesis), // '(' + TT(RightParenthesis), // ')' + NN, // '*' + PL, // '+' + TT(Comma), // ',' + DM, // '-' + NN, // '.' + NN, // '/' + + // 48 + DM, DM, DM, DM, DM, DM, DM, DM, DM, DM, // '0' - '9' + TT(Colon), // ':' + TT(Semicolon), // ';' + TT(LeftAngle), // '<' + TT(Equals), // '=' + TT(RightAngle), // '>' + NN, // '?' + + // 64 + NN, // '@' + ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'A' - 'M' + ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'N' - 'Z' + TT(LeftBracket), // '[' + NN, // '\' + TT(RightBracket), // ']' + NN, // '^' + ST, // '_' + + // 96 + NN, // '`' + + ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'a' - 'm' + ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'n' - 'z' + TT(LeftBrace), // '{' + NN, // '|' + TT(RightBrace), // '}' + NN, // '~' + NN, // '^?' non-printable + // 128 + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN}; #undef NN #undef BS @@ -144,153 +144,153 @@ namespace NYson { #undef PL #undef QU #undef TT - return static_cast<EReadStartCase>(lookupTable[static_cast<ui8>(ch)]); - } - - public: - TLexer(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit) - : TBase(blockStream, memoryLimit) - { - } - - void GetToken(TToken* token) { - char ch1 = TBase::SkipSpaceAndGetChar(); - auto state = GetStartState(ch1); - auto stateBits = static_cast<unsigned>(state); - - if (ch1 == '\0') { - *token = TToken::EndOfStream; - return; - } - - if (stateBits & 1) { // Other = x1b - if (stateBits & 1 << 1) { // Other = xxx11b - if (state == EReadStartCase::Quote) { - TStringBuf value; - TBase::Advance(1); - TBase::ReadQuotedString(&value); - *token = TToken(value); - } else if (state == EReadStartCase::DigitOrMinus) { - ReadNumeric<true>(token); - } else if (state == EReadStartCase::Plus) { - TBase::Advance(1); - - char ch2 = TBase::template GetChar<true>(); - - if (!isdigit(ch2)) { - *token = TToken(ETokenType::Plus); - } else { - ReadNumeric<true>(token); - } - } else if (state == EReadStartCase::String) { - TStringBuf value; - TBase::template ReadUnquotedString<true>(&value); - *token = TToken(value); - } else if (state == EReadStartCase::Percent) { - TBase::Advance(1); - char ch3 = TBase::template GetChar<true>(); - if (ch3 == 't' || ch3 == 'f') { - *token = TToken(TBase::template ReadBoolean<true>()); - } else { - *token = TToken(TBase::template ReadNanOrInf<true>()); - } - } else { // None - Y_ASSERT(state == EReadStartCase::None); + return static_cast<EReadStartCase>(lookupTable[static_cast<ui8>(ch)]); + } + + public: + TLexer(const TBlockStream& blockStream, TMaybe<ui64> memoryLimit) + : TBase(blockStream, memoryLimit) + { + } + + void GetToken(TToken* token) { + char ch1 = TBase::SkipSpaceAndGetChar(); + auto state = GetStartState(ch1); + auto stateBits = static_cast<unsigned>(state); + + if (ch1 == '\0') { + *token = TToken::EndOfStream; + return; + } + + if (stateBits & 1) { // Other = x1b + if (stateBits & 1 << 1) { // Other = xxx11b + if (state == EReadStartCase::Quote) { + TStringBuf value; + TBase::Advance(1); + TBase::ReadQuotedString(&value); + *token = TToken(value); + } else if (state == EReadStartCase::DigitOrMinus) { + ReadNumeric<true>(token); + } else if (state == EReadStartCase::Plus) { + TBase::Advance(1); + + char ch2 = TBase::template GetChar<true>(); + + if (!isdigit(ch2)) { + *token = TToken(ETokenType::Plus); + } else { + ReadNumeric<true>(token); + } + } else if (state == EReadStartCase::String) { + TStringBuf value; + TBase::template ReadUnquotedString<true>(&value); + *token = TToken(value); + } else if (state == EReadStartCase::Percent) { + TBase::Advance(1); + char ch3 = TBase::template GetChar<true>(); + if (ch3 == 't' || ch3 == 'f') { + *token = TToken(TBase::template ReadBoolean<true>()); + } else { + *token = TToken(TBase::template ReadNanOrInf<true>()); + } + } else { // None + Y_ASSERT(state == EReadStartCase::None); ythrow TYsonException() << "Unexpected " << ch1; - } - } else { // BinaryScalar = x01b - TBase::Advance(1); - if (state == EReadStartCase::BinaryDouble) { - double value; - TBase::ReadBinaryDouble(&value); - *token = TToken(value); - } else if (state == EReadStartCase::BinaryInt64) { - i64 value; - TBase::ReadBinaryInt64(&value); - *token = TToken(value); - } else if (state == EReadStartCase::BinaryUint64) { - ui64 value; - TBase::ReadBinaryUint64(&value); - *token = TToken(value); - } else if (state == EReadStartCase::BinaryFalse) { - *token = TToken(false); - } else if (state == EReadStartCase::BinaryTrue) { - *token = TToken(true); - } else { - Y_FAIL("unreachable"); - } + } + } else { // BinaryScalar = x01b + TBase::Advance(1); + if (state == EReadStartCase::BinaryDouble) { + double value; + TBase::ReadBinaryDouble(&value); + *token = TToken(value); + } else if (state == EReadStartCase::BinaryInt64) { + i64 value; + TBase::ReadBinaryInt64(&value); + *token = TToken(value); + } else if (state == EReadStartCase::BinaryUint64) { + ui64 value; + TBase::ReadBinaryUint64(&value); + *token = TToken(value); + } else if (state == EReadStartCase::BinaryFalse) { + *token = TToken(false); + } else if (state == EReadStartCase::BinaryTrue) { + *token = TToken(true); + } else { + Y_FAIL("unreachable"); + } } - } else { // BinaryStringOrOtherSpecialToken = x0b + } else { // BinaryStringOrOtherSpecialToken = x0b TBase::Advance(1); - if (stateBits & 1 << 1) { // OtherSpecialToken = 10b - Y_ASSERT((stateBits & 3) == static_cast<unsigned>(EReadStartCase::OtherSpecialToken)); - *token = TToken(ETokenType(stateBits >> 2)); - } else { // BinaryString = 00b - Y_ASSERT((stateBits & 3) == static_cast<unsigned>(EReadStartCase::BinaryString)); - TStringBuf value; - TBase::ReadBinaryString(&value); - *token = TToken(value); + if (stateBits & 1 << 1) { // OtherSpecialToken = 10b + Y_ASSERT((stateBits & 3) == static_cast<unsigned>(EReadStartCase::OtherSpecialToken)); + *token = TToken(ETokenType(stateBits >> 2)); + } else { // BinaryString = 00b + Y_ASSERT((stateBits & 3) == static_cast<unsigned>(EReadStartCase::BinaryString)); + TStringBuf value; + TBase::ReadBinaryString(&value); + *token = TToken(value); } } } - template <bool AllowFinish> - void ReadNumeric(TToken* token) { - TStringBuf valueBuffer; - ENumericResult numericResult = TBase::template ReadNumeric<AllowFinish>(&valueBuffer); + template <bool AllowFinish> + void ReadNumeric(TToken* token) { + TStringBuf valueBuffer; + ENumericResult numericResult = TBase::template ReadNumeric<AllowFinish>(&valueBuffer); - if (numericResult == ENumericResult::Double) { - try { - *token = TToken(FromString<double>(valueBuffer)); - } catch (yexception&) { + if (numericResult == ENumericResult::Double) { + try { + *token = TToken(FromString<double>(valueBuffer)); + } catch (yexception&) { ythrow TYsonException() << "Error parsing double literal " << valueBuffer; - } - } else if (numericResult == ENumericResult::Int64) { - try { - *token = TToken(FromString<i64>(valueBuffer)); - } catch (yexception&) { + } + } else if (numericResult == ENumericResult::Int64) { + try { + *token = TToken(FromString<i64>(valueBuffer)); + } catch (yexception&) { ythrow TYsonException() << "Error parsing int64 literal " << valueBuffer; - } - } else if (numericResult == ENumericResult::Uint64) { - try { - *token = TToken(FromString<ui64>(valueBuffer.SubStr(0, valueBuffer.size() - 1))); - } catch (yexception&) { + } + } else if (numericResult == ENumericResult::Uint64) { + try { + *token = TToken(FromString<ui64>(valueBuffer.SubStr(0, valueBuffer.size() - 1))); + } catch (yexception&) { ythrow TYsonException() << "Error parsing uint64 literal " << valueBuffer; - } - } + } + } } - }; - //////////////////////////////////////////////////////////////////////////////// - /*! \endinternal */ + }; + //////////////////////////////////////////////////////////////////////////////// + /*! \endinternal */ } - class TStatelessYsonLexerImplBase { - public: - virtual size_t GetToken(const TStringBuf& data, TToken* token) = 0; - - virtual ~TStatelessYsonLexerImplBase() { - } - }; - - template <bool EnableLinePositionInfo> - class TStatelesYsonLexerImpl: public TStatelessYsonLexerImplBase { - private: - using TLexer = NDetail::TLexer<TStringReader, EnableLinePositionInfo>; - TLexer Lexer; - - public: - TStatelesYsonLexerImpl() - : Lexer(TStringReader(), Nothing()) - { - } - - size_t GetToken(const TStringBuf& data, TToken* token) override { - Lexer.SetBuffer(data.begin(), data.end()); - Lexer.GetToken(token); - return Lexer.Begin() - data.begin(); - } - }; - - //////////////////////////////////////////////////////////////////////////////// + class TStatelessYsonLexerImplBase { + public: + virtual size_t GetToken(const TStringBuf& data, TToken* token) = 0; + + virtual ~TStatelessYsonLexerImplBase() { + } + }; + + template <bool EnableLinePositionInfo> + class TStatelesYsonLexerImpl: public TStatelessYsonLexerImplBase { + private: + using TLexer = NDetail::TLexer<TStringReader, EnableLinePositionInfo>; + TLexer Lexer; + + public: + TStatelesYsonLexerImpl() + : Lexer(TStringReader(), Nothing()) + { + } + + size_t GetToken(const TStringBuf& data, TToken* token) override { + Lexer.SetBuffer(data.begin(), data.end()); + Lexer.GetToken(token); + return Lexer.Begin() - data.begin(); + } + }; + + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/parser.cpp b/library/cpp/yson/parser.cpp index 783f9b9047..45f8d87014 100644 --- a/library/cpp/yson/parser.cpp +++ b/library/cpp/yson/parser.cpp @@ -7,173 +7,173 @@ #include <util/generic/buffer.h> namespace NYson { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - class TYsonParser::TImpl { - public: - TImpl( + class TYsonParser::TImpl { + public: + TImpl( NYT::NYson::IYsonConsumer* consumer, - IInputStream* stream, - EYsonType type, - bool enableLinePositionInfo, - TMaybe<ui64> memoryLimit = Nothing()) - : Consumer_(consumer) - , Stream_(stream) - , Type_(type) - , EnableLinePositionInfo_(enableLinePositionInfo) - , MemoryLimit_(memoryLimit) - { - } - - void Parse() { - TBuffer buffer(64 << 10); + IInputStream* stream, + EYsonType type, + bool enableLinePositionInfo, + TMaybe<ui64> memoryLimit = Nothing()) + : Consumer_(consumer) + , Stream_(stream) + , Type_(type) + , EnableLinePositionInfo_(enableLinePositionInfo) + , MemoryLimit_(memoryLimit) + { + } + + void Parse() { + TBuffer buffer(64 << 10); ParseYsonStreamImpl<NYT::NYson::IYsonConsumer, TStreamReader>( - TStreamReader(Stream_, buffer.Data(), buffer.Capacity()), - Consumer_, - Type_, - EnableLinePositionInfo_, - MemoryLimit_); - } - - private: + TStreamReader(Stream_, buffer.Data(), buffer.Capacity()), + Consumer_, + Type_, + EnableLinePositionInfo_, + MemoryLimit_); + } + + private: NYT::NYson::IYsonConsumer* Consumer_; - IInputStream* Stream_; - EYsonType Type_; - bool EnableLinePositionInfo_; - TMaybe<ui64> MemoryLimit_; - }; - - //////////////////////////////////////////////////////////////////////////////// - - TYsonParser::TYsonParser( + IInputStream* Stream_; + EYsonType Type_; + bool EnableLinePositionInfo_; + TMaybe<ui64> MemoryLimit_; + }; + + //////////////////////////////////////////////////////////////////////////////// + + TYsonParser::TYsonParser( NYT::NYson::IYsonConsumer* consumer, IInputStream* stream, EYsonType type, bool enableLinePositionInfo, - TMaybe<ui64> memoryLimit) - : Impl(new TImpl(consumer, stream, type, enableLinePositionInfo, memoryLimit)) + TMaybe<ui64> memoryLimit) + : Impl(new TImpl(consumer, stream, type, enableLinePositionInfo, memoryLimit)) { } - TYsonParser::~TYsonParser() { - } + TYsonParser::~TYsonParser() { + } - void TYsonParser::Parse() { - Impl->Parse(); - } + void TYsonParser::Parse() { + Impl->Parse(); + } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - class TStatelessYsonParser::TImpl { - private: - THolder<TStatelessYsonParserImplBase> Impl; + class TStatelessYsonParser::TImpl { + private: + THolder<TStatelessYsonParserImplBase> Impl; - public: - TImpl( + public: + TImpl( NYT::NYson::IYsonConsumer* consumer, - bool enableLinePositionInfo, - TMaybe<ui64> memoryLimit) - : Impl( - enableLinePositionInfo + bool enableLinePositionInfo, + TMaybe<ui64> memoryLimit) + : Impl( + enableLinePositionInfo ? static_cast<TStatelessYsonParserImplBase*>(new TStatelessYsonParserImpl<NYT::NYson::IYsonConsumer, true>(consumer, memoryLimit)) : static_cast<TStatelessYsonParserImplBase*>(new TStatelessYsonParserImpl<NYT::NYson::IYsonConsumer, false>(consumer, memoryLimit))) - { - } + { + } void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) { - Impl->Parse(data, type); - } - }; + Impl->Parse(data, type); + } + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - TStatelessYsonParser::TStatelessYsonParser( + TStatelessYsonParser::TStatelessYsonParser( NYT::NYson::IYsonConsumer* consumer, bool enableLinePositionInfo, TMaybe<ui64> memoryLimit) - : Impl(new TImpl(consumer, enableLinePositionInfo, memoryLimit)) - { - } - - TStatelessYsonParser::~TStatelessYsonParser() { - } - - void TStatelessYsonParser::Parse(const TStringBuf& data, EYsonType type) { + : Impl(new TImpl(consumer, enableLinePositionInfo, memoryLimit)) + { + } + + TStatelessYsonParser::~TStatelessYsonParser() { + } + + void TStatelessYsonParser::Parse(const TStringBuf& data, EYsonType type) { Impl->Parse(data, type); } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - void ParseYsonStringBuffer( - const TStringBuf& buffer, + void ParseYsonStringBuffer( + const TStringBuf& buffer, NYT::NYson::IYsonConsumer* consumer, - EYsonType type, - bool enableLinePositionInfo, - TMaybe<ui64> memoryLimit) { + EYsonType type, + bool enableLinePositionInfo, + TMaybe<ui64> memoryLimit) { ParseYsonStreamImpl<NYT::NYson::IYsonConsumer, TStringReader>( - TStringReader(buffer.begin(), buffer.end()), - consumer, - type, - enableLinePositionInfo, - memoryLimit); - } - - //////////////////////////////////////////////////////////////////////////////// - - class TYsonListParser::TImpl { - public: - TImpl( + TStringReader(buffer.begin(), buffer.end()), + consumer, + type, + enableLinePositionInfo, + memoryLimit); + } + + //////////////////////////////////////////////////////////////////////////////// + + class TYsonListParser::TImpl { + public: + TImpl( NYT::NYson::IYsonConsumer* consumer, - IInputStream* stream, - bool enableLinePositionInfo, - TMaybe<ui64> memoryLimit = Nothing()) - : Consumer_(consumer) - , Stream_(stream) - , EnableLinePositionInfo_(enableLinePositionInfo) - , MemoryLimit_(memoryLimit) - , Buffer_(64 << 10) - , Reader_(Stream_, Buffer_.Data(), Buffer_.Capacity()) - { - } - - bool Parse() { - if (!Impl_) { - Impl_.Reset( - EnableLinePositionInfo_ + IInputStream* stream, + bool enableLinePositionInfo, + TMaybe<ui64> memoryLimit = Nothing()) + : Consumer_(consumer) + , Stream_(stream) + , EnableLinePositionInfo_(enableLinePositionInfo) + , MemoryLimit_(memoryLimit) + , Buffer_(64 << 10) + , Reader_(Stream_, Buffer_.Data(), Buffer_.Capacity()) + { + } + + bool Parse() { + if (!Impl_) { + Impl_.Reset( + EnableLinePositionInfo_ ? static_cast<TYsonListParserImplBase*>(new TYsonListParserImpl<NYT::NYson::IYsonConsumer, TStreamReader, true>(Reader_, Consumer_, MemoryLimit_)) : static_cast<TYsonListParserImplBase*>(new TYsonListParserImpl<NYT::NYson::IYsonConsumer, TStreamReader, false>(Reader_, Consumer_, MemoryLimit_))); - } - return Impl_->Parse(); - } + } + return Impl_->Parse(); + } - private: + private: NYT::NYson::IYsonConsumer* Consumer_; - IInputStream* Stream_; - bool EnableLinePositionInfo_; - TMaybe<ui64> MemoryLimit_; - TBuffer Buffer_; - TStreamReader Reader_; - THolder<TYsonListParserImplBase> Impl_; - }; + IInputStream* Stream_; + bool EnableLinePositionInfo_; + TMaybe<ui64> MemoryLimit_; + TBuffer Buffer_; + TStreamReader Reader_; + THolder<TYsonListParserImplBase> Impl_; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - TYsonListParser::TYsonListParser( + TYsonListParser::TYsonListParser( NYT::NYson::IYsonConsumer* consumer, IInputStream* stream, bool enableLinePositionInfo, - TMaybe<ui64> memoryLimit) - : Impl(new TImpl(consumer, stream, enableLinePositionInfo, memoryLimit)) + TMaybe<ui64> memoryLimit) + : Impl(new TImpl(consumer, stream, enableLinePositionInfo, memoryLimit)) { } - TYsonListParser::~TYsonListParser() { - } + TYsonListParser::~TYsonListParser() { + } - bool TYsonListParser::Parse() { - return Impl->Parse(); - } + bool TYsonListParser::Parse() { + return Impl->Parse(); + } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/parser.h b/library/cpp/yson/parser.h index dce35a8cd4..f935840b98 100644 --- a/library/cpp/yson/parser.h +++ b/library/cpp/yson/parser.h @@ -12,72 +12,72 @@ struct IYsonConsumer; } // namespace NYT::NYson namespace NYson { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - class TYsonParser { - public: - TYsonParser( + class TYsonParser { + public: + TYsonParser( NYT::NYson::IYsonConsumer* consumer, - IInputStream* stream, + IInputStream* stream, EYsonType type = ::NYson::EYsonType::Node, - bool enableLinePositionInfo = false, - TMaybe<ui64> memoryLimit = Nothing()); + bool enableLinePositionInfo = false, + TMaybe<ui64> memoryLimit = Nothing()); - ~TYsonParser(); + ~TYsonParser(); - void Parse(); + void Parse(); - private: - class TImpl; - THolder<TImpl> Impl; - }; + private: + class TImpl; + THolder<TImpl> Impl; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - class TStatelessYsonParser { - public: - TStatelessYsonParser( + class TStatelessYsonParser { + public: + TStatelessYsonParser( NYT::NYson::IYsonConsumer* consumer, - bool enableLinePositionInfo = false, - TMaybe<ui64> memoryLimit = Nothing()); + bool enableLinePositionInfo = false, + TMaybe<ui64> memoryLimit = Nothing()); - ~TStatelessYsonParser(); + ~TStatelessYsonParser(); void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node); - private: - class TImpl; - THolder<TImpl> Impl; - }; + private: + class TImpl; + THolder<TImpl> Impl; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - class TYsonListParser { - public: - TYsonListParser( + class TYsonListParser { + public: + TYsonListParser( NYT::NYson::IYsonConsumer* consumer, - IInputStream* stream, - bool enableLinePositionInfo = false, - TMaybe<ui64> memoryLimit = Nothing()); + IInputStream* stream, + bool enableLinePositionInfo = false, + TMaybe<ui64> memoryLimit = Nothing()); - ~TYsonListParser(); + ~TYsonListParser(); - bool Parse(); // Returns false, if there is no more list items + bool Parse(); // Returns false, if there is no more list items - private: - class TImpl; - THolder<TImpl> Impl; - }; + private: + class TImpl; + THolder<TImpl> Impl; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - void ParseYsonStringBuffer( - const TStringBuf& buffer, + void ParseYsonStringBuffer( + const TStringBuf& buffer, NYT::NYson::IYsonConsumer* consumer, EYsonType type = ::NYson::EYsonType::Node, - bool enableLinePositionInfo = false, - TMaybe<ui64> memoryLimit = Nothing()); + bool enableLinePositionInfo = false, + TMaybe<ui64> memoryLimit = Nothing()); - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/parser_detail.h b/library/cpp/yson/parser_detail.h index 44223caf12..a4cfdc71da 100644 --- a/library/cpp/yson/parser_detail.h +++ b/library/cpp/yson/parser_detail.h @@ -3,379 +3,379 @@ #include "detail.h" namespace NYson { - namespace NDetail { - //////////////////////////////////////////////////////////////////////////////// - - template <class TConsumer, class TBlockStream, bool EnableLinePositionInfo> - class TParser - : public TLexerBase<TBlockStream, EnableLinePositionInfo> { - private: - using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>; - TConsumer* Consumer; - - public: - TParser(const TBlockStream& blockStream, TConsumer* consumer, TMaybe<ui64> memoryLimit) - : TBase(blockStream, memoryLimit) - , Consumer(consumer) - { - } - - void DoParse(EYsonType ysonType) { - switch (ysonType) { + namespace NDetail { + //////////////////////////////////////////////////////////////////////////////// + + template <class TConsumer, class TBlockStream, bool EnableLinePositionInfo> + class TParser + : public TLexerBase<TBlockStream, EnableLinePositionInfo> { + private: + using TBase = TLexerBase<TBlockStream, EnableLinePositionInfo>; + TConsumer* Consumer; + + public: + TParser(const TBlockStream& blockStream, TConsumer* consumer, TMaybe<ui64> memoryLimit) + : TBase(blockStream, memoryLimit) + , Consumer(consumer) + { + } + + void DoParse(EYsonType ysonType) { + switch (ysonType) { case ::NYson::EYsonType::Node: - ParseNode<true>(); - break; + ParseNode<true>(); + break; case ::NYson::EYsonType::ListFragment: - ParseListFragment<true>(EndSymbol); - break; + ParseListFragment<true>(EndSymbol); + break; case ::NYson::EYsonType::MapFragment: - ParseMapFragment<true>(EndSymbol); - break; + ParseMapFragment<true>(EndSymbol); + break; - default: - Y_FAIL("unreachable"); - } + default: + Y_FAIL("unreachable"); + } - while (!(TBase::IsFinished() && TBase::IsEmpty())) { - if (TBase::template SkipSpaceAndGetChar<true>() != EndSymbol) { + while (!(TBase::IsFinished() && TBase::IsEmpty())) { + if (TBase::template SkipSpaceAndGetChar<true>() != EndSymbol) { ythrow TYsonException() << "Stray '" << (*TBase::Begin()) << "' found"; - } else if (!TBase::IsEmpty()) { - TBase::Advance(1); - } - } - } - - bool DoParseListFragment(bool first) { - bool ret = first ? first : ParseListSeparator<true>(EndSymbol); - return ret && ParseListItem<true>(EndSymbol); - } - - void ParseAttributes() { - Consumer->OnBeginAttributes(); - ParseMapFragment(EndAttributesSymbol); - TBase::SkipCharToken(EndAttributesSymbol); - Consumer->OnEndAttributes(); - } - - void ParseMap() { - Consumer->OnBeginMap(); - ParseMapFragment(EndMapSymbol); - TBase::SkipCharToken(EndMapSymbol); - Consumer->OnEndMap(); - } - - void ParseList() { - Consumer->OnBeginList(); - ParseListFragment(EndListSymbol); - TBase::SkipCharToken(EndListSymbol); - Consumer->OnEndList(); + } else if (!TBase::IsEmpty()) { + TBase::Advance(1); + } + } + } + + bool DoParseListFragment(bool first) { + bool ret = first ? first : ParseListSeparator<true>(EndSymbol); + return ret && ParseListItem<true>(EndSymbol); } - template <bool AllowFinish> - void ParseNode() { - return ParseNode<AllowFinish>(TBase::SkipSpaceAndGetChar()); - } - - template <bool AllowFinish> - void ParseNode(char ch) { - if (ch == BeginAttributesSymbol) { - TBase::Advance(1); - ParseAttributes(); - ch = TBase::SkipSpaceAndGetChar(); - } - - switch (ch) { - case BeginMapSymbol: - TBase::Advance(1); - ParseMap(); - break; - - case BeginListSymbol: - TBase::Advance(1); - ParseList(); - break; - - case '"': { - TBase::Advance(1); - TStringBuf value; - TBase::ReadQuotedString(&value); - Consumer->OnStringScalar(value); - break; - } - case StringMarker: { - TBase::Advance(1); - TStringBuf value; - TBase::ReadBinaryString(&value); - Consumer->OnStringScalar(value); - break; - } - case Int64Marker: { - TBase::Advance(1); - i64 value; - TBase::ReadBinaryInt64(&value); - Consumer->OnInt64Scalar(value); - break; - } - case Uint64Marker: { - TBase::Advance(1); - ui64 value; - TBase::ReadBinaryUint64(&value); - Consumer->OnUint64Scalar(value); - break; - } - case DoubleMarker: { - TBase::Advance(1); - double value; - TBase::ReadBinaryDouble(&value); - Consumer->OnDoubleScalar(value); - break; - } - case FalseMarker: { - TBase::Advance(1); - Consumer->OnBooleanScalar(false); - break; - } - case TrueMarker: { - TBase::Advance(1); - Consumer->OnBooleanScalar(true); - break; - } - case EntitySymbol: - TBase::Advance(1); - Consumer->OnEntity(); - break; - - default: { - if (isdigit((unsigned char)ch) || ch == '-' || ch == '+') { // case of '+' is handled in AfterPlus state - ReadNumeric<AllowFinish>(); - } else if (isalpha((unsigned char)ch) || ch == '_') { - TStringBuf value; - TBase::template ReadUnquotedString<AllowFinish>(&value); - Consumer->OnStringScalar(value); - } else if (ch == '%') { - TBase::Advance(1); - ch = TBase::template GetChar<AllowFinish>(); - if (ch == 't' || ch == 'f') { - Consumer->OnBooleanScalar(TBase::template ReadBoolean<AllowFinish>()); - } else { - Consumer->OnDoubleScalar(TBase::template ReadNanOrInf<AllowFinish>()); - } - } else { + void ParseAttributes() { + Consumer->OnBeginAttributes(); + ParseMapFragment(EndAttributesSymbol); + TBase::SkipCharToken(EndAttributesSymbol); + Consumer->OnEndAttributes(); + } + + void ParseMap() { + Consumer->OnBeginMap(); + ParseMapFragment(EndMapSymbol); + TBase::SkipCharToken(EndMapSymbol); + Consumer->OnEndMap(); + } + + void ParseList() { + Consumer->OnBeginList(); + ParseListFragment(EndListSymbol); + TBase::SkipCharToken(EndListSymbol); + Consumer->OnEndList(); + } + + template <bool AllowFinish> + void ParseNode() { + return ParseNode<AllowFinish>(TBase::SkipSpaceAndGetChar()); + } + + template <bool AllowFinish> + void ParseNode(char ch) { + if (ch == BeginAttributesSymbol) { + TBase::Advance(1); + ParseAttributes(); + ch = TBase::SkipSpaceAndGetChar(); + } + + switch (ch) { + case BeginMapSymbol: + TBase::Advance(1); + ParseMap(); + break; + + case BeginListSymbol: + TBase::Advance(1); + ParseList(); + break; + + case '"': { + TBase::Advance(1); + TStringBuf value; + TBase::ReadQuotedString(&value); + Consumer->OnStringScalar(value); + break; + } + case StringMarker: { + TBase::Advance(1); + TStringBuf value; + TBase::ReadBinaryString(&value); + Consumer->OnStringScalar(value); + break; + } + case Int64Marker: { + TBase::Advance(1); + i64 value; + TBase::ReadBinaryInt64(&value); + Consumer->OnInt64Scalar(value); + break; + } + case Uint64Marker: { + TBase::Advance(1); + ui64 value; + TBase::ReadBinaryUint64(&value); + Consumer->OnUint64Scalar(value); + break; + } + case DoubleMarker: { + TBase::Advance(1); + double value; + TBase::ReadBinaryDouble(&value); + Consumer->OnDoubleScalar(value); + break; + } + case FalseMarker: { + TBase::Advance(1); + Consumer->OnBooleanScalar(false); + break; + } + case TrueMarker: { + TBase::Advance(1); + Consumer->OnBooleanScalar(true); + break; + } + case EntitySymbol: + TBase::Advance(1); + Consumer->OnEntity(); + break; + + default: { + if (isdigit((unsigned char)ch) || ch == '-' || ch == '+') { // case of '+' is handled in AfterPlus state + ReadNumeric<AllowFinish>(); + } else if (isalpha((unsigned char)ch) || ch == '_') { + TStringBuf value; + TBase::template ReadUnquotedString<AllowFinish>(&value); + Consumer->OnStringScalar(value); + } else if (ch == '%') { + TBase::Advance(1); + ch = TBase::template GetChar<AllowFinish>(); + if (ch == 't' || ch == 'f') { + Consumer->OnBooleanScalar(TBase::template ReadBoolean<AllowFinish>()); + } else { + Consumer->OnDoubleScalar(TBase::template ReadNanOrInf<AllowFinish>()); + } + } else { ythrow TYsonException() << "Unexpected '" << ch << "' while parsing node"; - } - } - } + } + } + } } - - void ParseKey() { - return ParseKey(TBase::SkipSpaceAndGetChar()); + + void ParseKey() { + return ParseKey(TBase::SkipSpaceAndGetChar()); } - - void ParseKey(char ch) { - switch (ch) { - case '"': { - TBase::Advance(1); - TStringBuf value; - TBase::ReadQuotedString(&value); - Consumer->OnKeyedItem(value); - break; - } - case StringMarker: { - TBase::Advance(1); - TStringBuf value; - TBase::ReadBinaryString(&value); - Consumer->OnKeyedItem(value); - break; - } - default: { - if (isalpha(ch) || ch == '_') { - TStringBuf value; - TBase::ReadUnquotedString(&value); - Consumer->OnKeyedItem(value); - } else { + + void ParseKey(char ch) { + switch (ch) { + case '"': { + TBase::Advance(1); + TStringBuf value; + TBase::ReadQuotedString(&value); + Consumer->OnKeyedItem(value); + break; + } + case StringMarker: { + TBase::Advance(1); + TStringBuf value; + TBase::ReadBinaryString(&value); + Consumer->OnKeyedItem(value); + break; + } + default: { + if (isalpha(ch) || ch == '_') { + TStringBuf value; + TBase::ReadUnquotedString(&value); + Consumer->OnKeyedItem(value); + } else { ythrow TYsonException() << "Unexpected '" << ch << "' while parsing key"; - } - } - } + } + } + } } - template <bool AllowFinish> - void ParseMapFragment(char endSymbol) { - char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); - while (ch != endSymbol) { - ParseKey(ch); - ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); - if (ch == KeyValueSeparatorSymbol) { - TBase::Advance(1); + template <bool AllowFinish> + void ParseMapFragment(char endSymbol) { + char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); + while (ch != endSymbol) { + ParseKey(ch); + ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); + if (ch == KeyValueSeparatorSymbol) { + TBase::Advance(1); } else { ythrow TYsonException() << "Expected '" << KeyValueSeparatorSymbol << "' but '" << ch << "' found"; } - ParseNode<AllowFinish>(); - ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); - if (ch == KeyedItemSeparatorSymbol) { - TBase::Advance(1); - ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); - } else if (ch != endSymbol) { + ParseNode<AllowFinish>(); + ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); + if (ch == KeyedItemSeparatorSymbol) { + TBase::Advance(1); + ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); + } else if (ch != endSymbol) { ythrow TYsonException() << "Expected '" << KeyedItemSeparatorSymbol << "' or '" << endSymbol << "' but '" << ch << "' found"; - } + } } } - void ParseMapFragment(char endSymbol) { - ParseMapFragment<false>(endSymbol); + void ParseMapFragment(char endSymbol) { + ParseMapFragment<false>(endSymbol); + } + + template <bool AllowFinish> + bool ParseListItem(char endSymbol) { + char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); + if (ch != endSymbol) { + Consumer->OnListItem(); + ParseNode<AllowFinish>(ch); + return true; + } + return false; } - - template <bool AllowFinish> - bool ParseListItem(char endSymbol) { - char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); - if (ch != endSymbol) { - Consumer->OnListItem(); - ParseNode<AllowFinish>(ch); - return true; - } - return false; - } - - template <bool AllowFinish> - bool ParseListSeparator(char endSymbol) { - char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); - if (ch == ListItemSeparatorSymbol) { - TBase::Advance(1); - return true; - } else if (ch != endSymbol) { + + template <bool AllowFinish> + bool ParseListSeparator(char endSymbol) { + char ch = TBase::template SkipSpaceAndGetChar<AllowFinish>(); + if (ch == ListItemSeparatorSymbol) { + TBase::Advance(1); + return true; + } else if (ch != endSymbol) { ythrow TYsonException() << "Expected '" << ListItemSeparatorSymbol << "' or '" << endSymbol << "' but '" << ch << "' found"; - } - return false; + } + return false; } - - template <bool AllowFinish> - void ParseListFragment(char endSymbol) { - while (ParseListItem<AllowFinish>(endSymbol) && ParseListSeparator<AllowFinish>(endSymbol)) { + + template <bool AllowFinish> + void ParseListFragment(char endSymbol) { + while (ParseListItem<AllowFinish>(endSymbol) && ParseListSeparator<AllowFinish>(endSymbol)) { } } - void ParseListFragment(char endSymbol) { - ParseListFragment<false>(endSymbol); + void ParseListFragment(char endSymbol) { + ParseListFragment<false>(endSymbol); } - - template <bool AllowFinish> - void ReadNumeric() { - TStringBuf valueBuffer; - ENumericResult numericResult = TBase::template ReadNumeric<AllowFinish>(&valueBuffer); - - if (numericResult == ENumericResult::Double) { - double value; - try { - value = FromString<double>(valueBuffer); - } catch (yexception& e) { - // This exception is wrapped in parser. + + template <bool AllowFinish> + void ReadNumeric() { + TStringBuf valueBuffer; + ENumericResult numericResult = TBase::template ReadNumeric<AllowFinish>(&valueBuffer); + + if (numericResult == ENumericResult::Double) { + double value; + try { + value = FromString<double>(valueBuffer); + } catch (yexception& e) { + // This exception is wrapped in parser. ythrow TYsonException() << "Failed to parse double literal '" << valueBuffer << "'" << e; - } - Consumer->OnDoubleScalar(value); - } else if (numericResult == ENumericResult::Int64) { - i64 value; - try { - value = FromString<i64>(valueBuffer); - } catch (yexception& e) { - // This exception is wrapped in parser. + } + Consumer->OnDoubleScalar(value); + } else if (numericResult == ENumericResult::Int64) { + i64 value; + try { + value = FromString<i64>(valueBuffer); + } catch (yexception& e) { + // This exception is wrapped in parser. ythrow TYsonException() << "Failed to parse int64 literal '" << valueBuffer << "'" << e; - } - Consumer->OnInt64Scalar(value); - } else if (numericResult == ENumericResult::Uint64) { - ui64 value; - try { - value = FromString<ui64>(valueBuffer.SubStr(0, valueBuffer.size() - 1)); - } catch (yexception& e) { - // This exception is wrapped in parser. + } + Consumer->OnInt64Scalar(value); + } else if (numericResult == ENumericResult::Uint64) { + ui64 value; + try { + value = FromString<ui64>(valueBuffer.SubStr(0, valueBuffer.size() - 1)); + } catch (yexception& e) { + // This exception is wrapped in parser. ythrow TYsonException() << "Failed to parse uint64 literal '" << valueBuffer << "'" << e; - } - Consumer->OnUint64Scalar(value); - } + } + Consumer->OnUint64Scalar(value); + } } - }; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// } - template <class TConsumer, class TBlockStream> - void ParseYsonStreamImpl( - const TBlockStream& blockStream, + template <class TConsumer, class TBlockStream> + void ParseYsonStreamImpl( + const TBlockStream& blockStream, NYT::NYson::IYsonConsumer* consumer, - EYsonType parsingMode, - bool enableLinePositionInfo, - TMaybe<ui64> memoryLimit) { - if (enableLinePositionInfo) { - using TImpl = NDetail::TParser<TConsumer, TBlockStream, true>; - TImpl impl(blockStream, consumer, memoryLimit); - impl.DoParse(parsingMode); - } else { - using TImpl = NDetail::TParser<TConsumer, TBlockStream, false>; - TImpl impl(blockStream, consumer, memoryLimit); - impl.DoParse(parsingMode); + EYsonType parsingMode, + bool enableLinePositionInfo, + TMaybe<ui64> memoryLimit) { + if (enableLinePositionInfo) { + using TImpl = NDetail::TParser<TConsumer, TBlockStream, true>; + TImpl impl(blockStream, consumer, memoryLimit); + impl.DoParse(parsingMode); + } else { + using TImpl = NDetail::TParser<TConsumer, TBlockStream, false>; + TImpl impl(blockStream, consumer, memoryLimit); + impl.DoParse(parsingMode); } } - class TStatelessYsonParserImplBase { - public: + class TStatelessYsonParserImplBase { + public: virtual void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) = 0; - virtual ~TStatelessYsonParserImplBase() { - } - }; - - template <class TConsumer, bool EnableLinePositionInfo> - class TStatelessYsonParserImpl - : public TStatelessYsonParserImplBase { - private: - using TParser = NDetail::TParser<TConsumer, TStringReader, EnableLinePositionInfo>; - TParser Parser; - - public: - TStatelessYsonParserImpl(TConsumer* consumer, TMaybe<ui64> memoryLimit) - : Parser(TStringReader(), consumer, memoryLimit) - { + virtual ~TStatelessYsonParserImplBase() { } + }; - void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) override { - Parser.SetBuffer(data.begin(), data.end()); - Parser.DoParse(type); - } - }; - - class TYsonListParserImplBase { - public: - virtual bool Parse() = 0; + template <class TConsumer, bool EnableLinePositionInfo> + class TStatelessYsonParserImpl + : public TStatelessYsonParserImplBase { + private: + using TParser = NDetail::TParser<TConsumer, TStringReader, EnableLinePositionInfo>; + TParser Parser; - virtual ~TYsonListParserImplBase() { - } - }; - - template <class TConsumer, class TBlockStream, bool EnableLinePositionInfo> - class TYsonListParserImpl - : public TYsonListParserImplBase { - private: - using TParser = NDetail::TParser<TConsumer, TBlockStream, EnableLinePositionInfo>; - TParser Parser; - bool First = true; - - public: - TYsonListParserImpl(const TBlockStream& blockStream, TConsumer* consumer, TMaybe<ui64> memoryLimit) - : Parser(blockStream, consumer, memoryLimit) - { - } + public: + TStatelessYsonParserImpl(TConsumer* consumer, TMaybe<ui64> memoryLimit) + : Parser(TStringReader(), consumer, memoryLimit) + { + } - bool Parse() override { - bool ret = Parser.DoParseListFragment(First); - First = false; - return ret; + void Parse(const TStringBuf& data, EYsonType type = ::NYson::EYsonType::Node) override { + Parser.SetBuffer(data.begin(), data.end()); + Parser.DoParse(type); } - }; - - //////////////////////////////////////////////////////////////////////////////// + }; + + class TYsonListParserImplBase { + public: + virtual bool Parse() = 0; + + virtual ~TYsonListParserImplBase() { + } + }; + + template <class TConsumer, class TBlockStream, bool EnableLinePositionInfo> + class TYsonListParserImpl + : public TYsonListParserImplBase { + private: + using TParser = NDetail::TParser<TConsumer, TBlockStream, EnableLinePositionInfo>; + TParser Parser; + bool First = true; + + public: + TYsonListParserImpl(const TBlockStream& blockStream, TConsumer* consumer, TMaybe<ui64> memoryLimit) + : Parser(blockStream, consumer, memoryLimit) + { + } + + bool Parse() override { + bool ret = Parser.DoParseListFragment(First); + First = false; + return ret; + } + }; + + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/public.h b/library/cpp/yson/public.h index 1ed793592b..c03e53d41f 100644 --- a/library/cpp/yson/public.h +++ b/library/cpp/yson/public.h @@ -8,7 +8,7 @@ namespace NYson { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// using NYT::NYson::EYsonFormat; using NYT::NYson::EYsonType; @@ -22,9 +22,9 @@ namespace NYson { class TStatelessYsonParser; class TYsonListParser; - class TYsonException - : public yexception {}; + class TYsonException + : public yexception {}; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/token.cpp b/library/cpp/yson/token.cpp index c8584c8c2e..13c6f170bb 100644 --- a/library/cpp/yson/token.cpp +++ b/library/cpp/yson/token.cpp @@ -4,233 +4,233 @@ #include <util/string/printf.h> namespace NYson { - //////////////////////////////////////////////////////////////////////////////// - - ETokenType CharToTokenType(char ch) { - switch (ch) { - case ';': - return ETokenType::Semicolon; - case '=': - return ETokenType::Equals; - case '{': - return ETokenType::LeftBrace; - case '}': - return ETokenType::RightBrace; - case '#': - return ETokenType::Hash; - case '[': - return ETokenType::LeftBracket; - case ']': - return ETokenType::RightBracket; - case '<': - return ETokenType::LeftAngle; - case '>': - return ETokenType::RightAngle; - case '(': - return ETokenType::LeftParenthesis; - case ')': - return ETokenType::RightParenthesis; - case '+': - return ETokenType::Plus; - case ':': - return ETokenType::Colon; - case ',': - return ETokenType::Comma; - default: - return ETokenType::EndOfStream; - } - } - - char TokenTypeToChar(ETokenType type) { - switch (type) { - case ETokenType::Semicolon: - return ';'; - case ETokenType::Equals: - return '='; - case ETokenType::Hash: - return '#'; - case ETokenType::LeftBracket: - return '['; - case ETokenType::RightBracket: - return ']'; - case ETokenType::LeftBrace: - return '{'; - case ETokenType::RightBrace: - return '}'; - case ETokenType::LeftAngle: - return '<'; - case ETokenType::RightAngle: - return '>'; - case ETokenType::LeftParenthesis: - return '('; - case ETokenType::RightParenthesis: - return ')'; - case ETokenType::Plus: - return '+'; - case ETokenType::Colon: - return ':'; - case ETokenType::Comma: - return ','; - default: - Y_FAIL("unreachable"); - } - } - - TString TokenTypeToString(ETokenType type) { + //////////////////////////////////////////////////////////////////////////////// + + ETokenType CharToTokenType(char ch) { + switch (ch) { + case ';': + return ETokenType::Semicolon; + case '=': + return ETokenType::Equals; + case '{': + return ETokenType::LeftBrace; + case '}': + return ETokenType::RightBrace; + case '#': + return ETokenType::Hash; + case '[': + return ETokenType::LeftBracket; + case ']': + return ETokenType::RightBracket; + case '<': + return ETokenType::LeftAngle; + case '>': + return ETokenType::RightAngle; + case '(': + return ETokenType::LeftParenthesis; + case ')': + return ETokenType::RightParenthesis; + case '+': + return ETokenType::Plus; + case ':': + return ETokenType::Colon; + case ',': + return ETokenType::Comma; + default: + return ETokenType::EndOfStream; + } + } + + char TokenTypeToChar(ETokenType type) { + switch (type) { + case ETokenType::Semicolon: + return ';'; + case ETokenType::Equals: + return '='; + case ETokenType::Hash: + return '#'; + case ETokenType::LeftBracket: + return '['; + case ETokenType::RightBracket: + return ']'; + case ETokenType::LeftBrace: + return '{'; + case ETokenType::RightBrace: + return '}'; + case ETokenType::LeftAngle: + return '<'; + case ETokenType::RightAngle: + return '>'; + case ETokenType::LeftParenthesis: + return '('; + case ETokenType::RightParenthesis: + return ')'; + case ETokenType::Plus: + return '+'; + case ETokenType::Colon: + return ':'; + case ETokenType::Comma: + return ','; + default: + Y_FAIL("unreachable"); + } + } + + TString TokenTypeToString(ETokenType type) { return TString(1, TokenTypeToChar(type)); } - //////////////////////////////////////////////////////////////////////////////// - - const TToken TToken::EndOfStream; - - TToken::TToken() - : Type_(ETokenType::EndOfStream) - , Int64Value(0) - , Uint64Value(0) - , DoubleValue(0.0) - , BooleanValue(false) - { - } - - TToken::TToken(ETokenType type) - : Type_(type) - , Int64Value(0) - , Uint64Value(0) - , DoubleValue(0.0) - , BooleanValue(false) - { - switch (type) { - case ETokenType::String: - case ETokenType::Int64: - case ETokenType::Uint64: - case ETokenType::Double: - case ETokenType::Boolean: - Y_FAIL("unreachable"); - default: - break; - } - } - - TToken::TToken(const TStringBuf& stringValue) - : Type_(ETokenType::String) - , StringValue(stringValue) - , Int64Value(0) - , Uint64Value(0) - , DoubleValue(0.0) - , BooleanValue(false) - { - } - - TToken::TToken(i64 int64Value) - : Type_(ETokenType::Int64) - , Int64Value(int64Value) - , Uint64Value(0) - , DoubleValue(0.0) - { - } - - TToken::TToken(ui64 uint64Value) - : Type_(ETokenType::Uint64) - , Int64Value(0) - , Uint64Value(uint64Value) - , DoubleValue(0.0) - , BooleanValue(false) - { - } - - TToken::TToken(double doubleValue) - : Type_(ETokenType::Double) - , Int64Value(0) - , Uint64Value(0) - , DoubleValue(doubleValue) - , BooleanValue(false) - { - } - - TToken::TToken(bool booleanValue) - : Type_(ETokenType::Boolean) - , Int64Value(0) - , DoubleValue(0.0) - , BooleanValue(booleanValue) - { - } - - bool TToken::IsEmpty() const { - return Type_ == ETokenType::EndOfStream; - } - - const TStringBuf& TToken::GetStringValue() const { - CheckType(ETokenType::String); - return StringValue; - } - - i64 TToken::GetInt64Value() const { - CheckType(ETokenType::Int64); - return Int64Value; - } - - ui64 TToken::GetUint64Value() const { - CheckType(ETokenType::Uint64); - return Uint64Value; - } - - double TToken::GetDoubleValue() const { - CheckType(ETokenType::Double); - return DoubleValue; - } - - bool TToken::GetBooleanValue() const { - CheckType(ETokenType::Boolean); - return BooleanValue; - } - - void TToken::CheckType(ETokenType expectedType) const { - if (Type_ != expectedType) { - if (Type_ == ETokenType::EndOfStream) { + //////////////////////////////////////////////////////////////////////////////// + + const TToken TToken::EndOfStream; + + TToken::TToken() + : Type_(ETokenType::EndOfStream) + , Int64Value(0) + , Uint64Value(0) + , DoubleValue(0.0) + , BooleanValue(false) + { + } + + TToken::TToken(ETokenType type) + : Type_(type) + , Int64Value(0) + , Uint64Value(0) + , DoubleValue(0.0) + , BooleanValue(false) + { + switch (type) { + case ETokenType::String: + case ETokenType::Int64: + case ETokenType::Uint64: + case ETokenType::Double: + case ETokenType::Boolean: + Y_FAIL("unreachable"); + default: + break; + } + } + + TToken::TToken(const TStringBuf& stringValue) + : Type_(ETokenType::String) + , StringValue(stringValue) + , Int64Value(0) + , Uint64Value(0) + , DoubleValue(0.0) + , BooleanValue(false) + { + } + + TToken::TToken(i64 int64Value) + : Type_(ETokenType::Int64) + , Int64Value(int64Value) + , Uint64Value(0) + , DoubleValue(0.0) + { + } + + TToken::TToken(ui64 uint64Value) + : Type_(ETokenType::Uint64) + , Int64Value(0) + , Uint64Value(uint64Value) + , DoubleValue(0.0) + , BooleanValue(false) + { + } + + TToken::TToken(double doubleValue) + : Type_(ETokenType::Double) + , Int64Value(0) + , Uint64Value(0) + , DoubleValue(doubleValue) + , BooleanValue(false) + { + } + + TToken::TToken(bool booleanValue) + : Type_(ETokenType::Boolean) + , Int64Value(0) + , DoubleValue(0.0) + , BooleanValue(booleanValue) + { + } + + bool TToken::IsEmpty() const { + return Type_ == ETokenType::EndOfStream; + } + + const TStringBuf& TToken::GetStringValue() const { + CheckType(ETokenType::String); + return StringValue; + } + + i64 TToken::GetInt64Value() const { + CheckType(ETokenType::Int64); + return Int64Value; + } + + ui64 TToken::GetUint64Value() const { + CheckType(ETokenType::Uint64); + return Uint64Value; + } + + double TToken::GetDoubleValue() const { + CheckType(ETokenType::Double); + return DoubleValue; + } + + bool TToken::GetBooleanValue() const { + CheckType(ETokenType::Boolean); + return BooleanValue; + } + + void TToken::CheckType(ETokenType expectedType) const { + if (Type_ != expectedType) { + if (Type_ == ETokenType::EndOfStream) { ythrow TYsonException() << "Unexpected end of stream (ExpectedType: " << TokenTypeToString(expectedType) << ")"; - } else { + } else { ythrow TYsonException() << "Unexpected token (Token: '" << ToString(*this) << "', Type: " << TokenTypeToString(Type_) << ", ExpectedType: " << TokenTypeToString(expectedType) << ")"; - } + } } } - void TToken::Reset() { - Type_ = ETokenType::EndOfStream; - Int64Value = 0; - Uint64Value = 0; - DoubleValue = 0.0; - StringValue = TStringBuf(); - BooleanValue = false; - } + void TToken::Reset() { + Type_ = ETokenType::EndOfStream; + Int64Value = 0; + Uint64Value = 0; + DoubleValue = 0.0; + StringValue = TStringBuf(); + BooleanValue = false; + } - TString ToString(const TToken& token) { - switch (token.GetType()) { - case ETokenType::EndOfStream: - return TString(); + TString ToString(const TToken& token) { + switch (token.GetType()) { + case ETokenType::EndOfStream: + return TString(); - case ETokenType::String: - return TString(token.GetStringValue()); + case ETokenType::String: + return TString(token.GetStringValue()); - case ETokenType::Int64: - return ::ToString(token.GetInt64Value()); + case ETokenType::Int64: + return ::ToString(token.GetInt64Value()); - case ETokenType::Uint64: - return ::ToString(token.GetUint64Value()); + case ETokenType::Uint64: + return ::ToString(token.GetUint64Value()); - case ETokenType::Double: - return ::ToString(token.GetDoubleValue()); + case ETokenType::Double: + return ::ToString(token.GetDoubleValue()); - case ETokenType::Boolean: - return token.GetBooleanValue() ? "true" : "false"; + case ETokenType::Boolean: + return token.GetBooleanValue() ? "true" : "false"; - default: - return TokenTypeToString(token.GetType()); - } + default: + return TokenTypeToString(token.GetType()); + } } - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/token.h b/library/cpp/yson/token.h index 7283e56950..a0a3975eff 100644 --- a/library/cpp/yson/token.h +++ b/library/cpp/yson/token.h @@ -5,89 +5,89 @@ #include <util/generic/strbuf.h> namespace NYson { - //////////////////////////////////////////////////////////////////////////////// - - enum ETokenType { - EndOfStream, - - String, - Int64, - Uint64, - Double, - Boolean, - - // Special values: - // YSON - Semicolon, // ; - Equals, // = - Hash, // # - LeftBracket, // [ - RightBracket, // ] - LeftBrace, // { - RightBrace, // } - LeftAngle, // < - RightAngle, // > - - // Table ranges - LeftParenthesis, // ( - RightParenthesis, // ) - Plus, // + - Colon, // : - Comma, // , - }; - - //////////////////////////////////////////////////////////////////////////////// - - ETokenType CharToTokenType(char ch); - char TokenTypeToChar(ETokenType type); - TString TokenTypeToString(ETokenType type); - - //////////////////////////////////////////////////////////////////////////////// - - class TLexerImpl; - - //////////////////////////////////////////////////////////////////////////////// - - class TToken { - public: - static const TToken EndOfStream; - - TToken(); - TToken(ETokenType type); - explicit TToken(const TStringBuf& stringValue); - explicit TToken(i64 int64Value); - explicit TToken(ui64 int64Value); - explicit TToken(double doubleValue); - explicit TToken(bool booleanValue); - - ETokenType GetType() const { - return Type_; - } - - bool IsEmpty() const; - const TStringBuf& GetStringValue() const; - i64 GetInt64Value() const; - ui64 GetUint64Value() const; - double GetDoubleValue() const; - bool GetBooleanValue() const; - - void CheckType(ETokenType expectedType) const; - void Reset(); - - private: - friend class TLexerImpl; - - ETokenType Type_; - - TStringBuf StringValue; - i64 Int64Value; - ui64 Uint64Value; - double DoubleValue; - bool BooleanValue; - }; - - TString ToString(const TToken& token); - - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// + + enum ETokenType { + EndOfStream, + + String, + Int64, + Uint64, + Double, + Boolean, + + // Special values: + // YSON + Semicolon, // ; + Equals, // = + Hash, // # + LeftBracket, // [ + RightBracket, // ] + LeftBrace, // { + RightBrace, // } + LeftAngle, // < + RightAngle, // > + + // Table ranges + LeftParenthesis, // ( + RightParenthesis, // ) + Plus, // + + Colon, // : + Comma, // , + }; + + //////////////////////////////////////////////////////////////////////////////// + + ETokenType CharToTokenType(char ch); + char TokenTypeToChar(ETokenType type); + TString TokenTypeToString(ETokenType type); + + //////////////////////////////////////////////////////////////////////////////// + + class TLexerImpl; + + //////////////////////////////////////////////////////////////////////////////// + + class TToken { + public: + static const TToken EndOfStream; + + TToken(); + TToken(ETokenType type); + explicit TToken(const TStringBuf& stringValue); + explicit TToken(i64 int64Value); + explicit TToken(ui64 int64Value); + explicit TToken(double doubleValue); + explicit TToken(bool booleanValue); + + ETokenType GetType() const { + return Type_; + } + + bool IsEmpty() const; + const TStringBuf& GetStringValue() const; + i64 GetInt64Value() const; + ui64 GetUint64Value() const; + double GetDoubleValue() const; + bool GetBooleanValue() const; + + void CheckType(ETokenType expectedType) const; + void Reset(); + + private: + friend class TLexerImpl; + + ETokenType Type_; + + TStringBuf StringValue; + i64 Int64Value; + ui64 Uint64Value; + double DoubleValue; + bool BooleanValue; + }; + + TString ToString(const TToken& token); + + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/tokenizer.cpp b/library/cpp/yson/tokenizer.cpp index 06760170d4..e0296e8ea7 100644 --- a/library/cpp/yson/tokenizer.cpp +++ b/library/cpp/yson/tokenizer.cpp @@ -1,37 +1,37 @@ #include "tokenizer.h" namespace NYson { - //////////////////////////////////////////////////////////////////////////////// - - TTokenizer::TTokenizer(const TStringBuf& input) - : Input(input) - , Parsed(0) - { - } - - bool TTokenizer::ParseNext() { - Input = Input.Tail(Parsed); - Token.Reset(); - Parsed = Lexer.GetToken(Input, &Token); - return !CurrentToken().IsEmpty(); - } - - const TToken& TTokenizer::CurrentToken() const { - return Token; - } - - ETokenType TTokenizer::GetCurrentType() const { - return CurrentToken().GetType(); - } - - TStringBuf TTokenizer::GetCurrentSuffix() const { - return Input.Tail(Parsed); - } - - const TStringBuf& TTokenizer::CurrentInput() const { - return Input; - } - - //////////////////////////////////////////////////////////////////////////////// - + //////////////////////////////////////////////////////////////////////////////// + + TTokenizer::TTokenizer(const TStringBuf& input) + : Input(input) + , Parsed(0) + { + } + + bool TTokenizer::ParseNext() { + Input = Input.Tail(Parsed); + Token.Reset(); + Parsed = Lexer.GetToken(Input, &Token); + return !CurrentToken().IsEmpty(); + } + + const TToken& TTokenizer::CurrentToken() const { + return Token; + } + + ETokenType TTokenizer::GetCurrentType() const { + return CurrentToken().GetType(); + } + + TStringBuf TTokenizer::GetCurrentSuffix() const { + return Input.Tail(Parsed); + } + + const TStringBuf& TTokenizer::CurrentInput() const { + return Input; + } + + //////////////////////////////////////////////////////////////////////////////// + } // namespace NYson diff --git a/library/cpp/yson/tokenizer.h b/library/cpp/yson/tokenizer.h index 0576aace95..c4449f590c 100644 --- a/library/cpp/yson/tokenizer.h +++ b/library/cpp/yson/tokenizer.h @@ -4,25 +4,25 @@ #include "lexer.h" namespace NYson { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - class TTokenizer { - public: - explicit TTokenizer(const TStringBuf& input); + class TTokenizer { + public: + explicit TTokenizer(const TStringBuf& input); - bool ParseNext(); - const TToken& CurrentToken() const; - ETokenType GetCurrentType() const; - TStringBuf GetCurrentSuffix() const; - const TStringBuf& CurrentInput() const; + bool ParseNext(); + const TToken& CurrentToken() const; + ETokenType GetCurrentType() const; + TStringBuf GetCurrentSuffix() const; + const TStringBuf& CurrentInput() const; - private: - TStringBuf Input; - TToken Token; - TStatelessLexer Lexer; - size_t Parsed; - }; + private: + TStringBuf Input; + TToken Token; + TStatelessLexer Lexer; + size_t Parsed; + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/varint.cpp b/library/cpp/yson/varint.cpp index d538ee3cff..d715d08294 100644 --- a/library/cpp/yson/varint.cpp +++ b/library/cpp/yson/varint.cpp @@ -5,67 +5,67 @@ #include <util/generic/yexception.h> namespace NYson { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - int WriteVarUInt64(IOutputStream* output, ui64 value) { - bool stop = false; - int bytesWritten = 0; - while (!stop) { - ++bytesWritten; - ui8 byte = static_cast<ui8>(value | 0x80); - value >>= 7; - if (value == 0) { - stop = true; - byte &= 0x7F; - } - output->Write(byte); + int WriteVarUInt64(IOutputStream* output, ui64 value) { + bool stop = false; + int bytesWritten = 0; + while (!stop) { + ++bytesWritten; + ui8 byte = static_cast<ui8>(value | 0x80); + value >>= 7; + if (value == 0) { + stop = true; + byte &= 0x7F; + } + output->Write(byte); } - return bytesWritten; + return bytesWritten; } - int WriteVarInt32(IOutputStream* output, i32 value) { - return WriteVarUInt64(output, static_cast<ui64>(ZigZagEncode32(value))); - } - - int WriteVarInt64(IOutputStream* output, i64 value) { - return WriteVarUInt64(output, static_cast<ui64>(ZigZagEncode64(value))); - } + int WriteVarInt32(IOutputStream* output, i32 value) { + return WriteVarUInt64(output, static_cast<ui64>(ZigZagEncode32(value))); + } - int ReadVarUInt64(IInputStream* input, ui64* value) { - size_t count = 0; - ui64 result = 0; + int WriteVarInt64(IOutputStream* output, i64 value) { + return WriteVarUInt64(output, static_cast<ui64>(ZigZagEncode64(value))); + } - ui8 byte = 0; - do { - if (7 * count > 8 * sizeof(ui64)) { - ythrow yexception() << "The data is too long to read ui64"; - } - if (input->Read(&byte, 1) != 1) { - ythrow yexception() << "The data is too long to read ui64"; - } - result |= (static_cast<ui64>(byte & 0x7F)) << (7 * count); - ++count; - } while (byte & 0x80); + int ReadVarUInt64(IInputStream* input, ui64* value) { + size_t count = 0; + ui64 result = 0; - *value = result; - return count; - } - - int ReadVarInt32(IInputStream* input, i32* value) { - ui64 varInt; - int bytesRead = ReadVarUInt64(input, &varInt); - if (varInt > Max<ui32>()) { + ui8 byte = 0; + do { + if (7 * count > 8 * sizeof(ui64)) { + ythrow yexception() << "The data is too long to read ui64"; + } + if (input->Read(&byte, 1) != 1) { + ythrow yexception() << "The data is too long to read ui64"; + } + result |= (static_cast<ui64>(byte & 0x7F)) << (7 * count); + ++count; + } while (byte & 0x80); + + *value = result; + return count; + } + + int ReadVarInt32(IInputStream* input, i32* value) { + ui64 varInt; + int bytesRead = ReadVarUInt64(input, &varInt); + if (varInt > Max<ui32>()) { ythrow yexception() << "The data is too long to read ui64"; } - *value = ZigZagDecode32(static_cast<ui32>(varInt)); - return bytesRead; - } + *value = ZigZagDecode32(static_cast<ui32>(varInt)); + return bytesRead; + } - int ReadVarInt64(IInputStream* input, i64* value) { - ui64 varInt; - int bytesRead = ReadVarUInt64(input, &varInt); - *value = ZigZagDecode64(varInt); - return bytesRead; + int ReadVarInt64(IInputStream* input, i64* value) { + ui64 varInt; + int bytesRead = ReadVarUInt64(input, &varInt); + *value = ZigZagDecode64(varInt); + return bytesRead; } } // namespace NYson diff --git a/library/cpp/yson/varint.h b/library/cpp/yson/varint.h index 80b1184e57..3733bbfc8c 100644 --- a/library/cpp/yson/varint.h +++ b/library/cpp/yson/varint.h @@ -5,20 +5,20 @@ #include <util/system/defaults.h> namespace NYson { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - // Various functions that read/write varints from/to a stream. + // Various functions that read/write varints from/to a stream. - // Returns the number of bytes written. - int WriteVarUInt64(IOutputStream* output, ui64 value); - int WriteVarInt32(IOutputStream* output, i32 value); - int WriteVarInt64(IOutputStream* output, i64 value); + // Returns the number of bytes written. + int WriteVarUInt64(IOutputStream* output, ui64 value); + int WriteVarInt32(IOutputStream* output, i32 value); + int WriteVarInt64(IOutputStream* output, i64 value); - // Returns the number of bytes read. - int ReadVarUInt64(IInputStream* input, ui64* value); - int ReadVarInt32(IInputStream* input, i32* value); - int ReadVarInt64(IInputStream* input, i64* value); + // Returns the number of bytes read. + int ReadVarUInt64(IInputStream* input, ui64* value); + int ReadVarInt32(IInputStream* input, i32* value); + int ReadVarInt64(IInputStream* input, i64* value); - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/writer.cpp b/library/cpp/yson/writer.cpp index 054459f9f5..4810ca14d8 100644 --- a/library/cpp/yson/writer.cpp +++ b/library/cpp/yson/writer.cpp @@ -11,345 +11,345 @@ #include <cmath> namespace NYson { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - // Copied from <util/string/escape.cpp> - namespace { + // Copied from <util/string/escape.cpp> + namespace { inline char HexDigit(char value) { - Y_ASSERT(value < 16); - if (value < 10) - return '0' + value; - else - return 'A' + value - 10; - } + Y_ASSERT(value < 16); + if (value < 10) + return '0' + value; + else + return 'A' + value - 10; + } inline char OctDigit(char value) { - Y_ASSERT(value < 8); - return '0' + value; - } + Y_ASSERT(value < 8); + return '0' + value; + } inline bool IsPrintable(char c) { - return c >= 32 && c <= 126; - } + return c >= 32 && c <= 126; + } inline bool IsHexDigit(char c) { - return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); - } + return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); + } inline bool IsOctDigit(char c) { - return c >= '0' && c <= '7'; - } + return c >= '0' && c <= '7'; + } const size_t ESCAPE_C_BUFFER_SIZE = 4; inline size_t EscapeC(unsigned char c, char next, char r[ESCAPE_C_BUFFER_SIZE]) { - // (1) Printable characters go as-is, except backslash and double quote. - // (2) Characters \r, \n, \t and \0 ... \7 replaced by their simple escape characters (if possible). - // (3) Otherwise, character is encoded using hexadecimal escape sequence (if possible), or octal. - if (c == '\"') { - r[0] = '\\'; - r[1] = '\"'; - return 2; - } else if (c == '\\') { - r[0] = '\\'; - r[1] = '\\'; - return 2; - } else if (IsPrintable(c)) { - r[0] = c; - return 1; - } else if (c == '\r') { - r[0] = '\\'; - r[1] = 'r'; - return 2; - } else if (c == '\n') { - r[0] = '\\'; - r[1] = 'n'; - return 2; - } else if (c == '\t') { - r[0] = '\\'; - r[1] = 't'; - return 2; - } else if (c < 8 && !IsOctDigit(next)) { - r[0] = '\\'; - r[1] = OctDigit(c); - return 2; - } else if (!IsHexDigit(next)) { - r[0] = '\\'; - r[1] = 'x'; - r[2] = HexDigit((c & 0xF0) >> 4); - r[3] = HexDigit((c & 0x0F) >> 0); - return 4; - } else { - r[0] = '\\'; - r[1] = OctDigit((c & 0700) >> 6); - r[2] = OctDigit((c & 0070) >> 3); - r[3] = OctDigit((c & 0007) >> 0); - return 4; - } + // (1) Printable characters go as-is, except backslash and double quote. + // (2) Characters \r, \n, \t and \0 ... \7 replaced by their simple escape characters (if possible). + // (3) Otherwise, character is encoded using hexadecimal escape sequence (if possible), or octal. + if (c == '\"') { + r[0] = '\\'; + r[1] = '\"'; + return 2; + } else if (c == '\\') { + r[0] = '\\'; + r[1] = '\\'; + return 2; + } else if (IsPrintable(c)) { + r[0] = c; + return 1; + } else if (c == '\r') { + r[0] = '\\'; + r[1] = 'r'; + return 2; + } else if (c == '\n') { + r[0] = '\\'; + r[1] = 'n'; + return 2; + } else if (c == '\t') { + r[0] = '\\'; + r[1] = 't'; + return 2; + } else if (c < 8 && !IsOctDigit(next)) { + r[0] = '\\'; + r[1] = OctDigit(c); + return 2; + } else if (!IsHexDigit(next)) { + r[0] = '\\'; + r[1] = 'x'; + r[2] = HexDigit((c & 0xF0) >> 4); + r[3] = HexDigit((c & 0x0F) >> 0); + return 4; + } else { + r[0] = '\\'; + r[1] = OctDigit((c & 0700) >> 6); + r[2] = OctDigit((c & 0070) >> 3); + r[3] = OctDigit((c & 0007) >> 0); + return 4; + } + } + + void EscapeC(const char* str, size_t len, IOutputStream& output) { + char buffer[ESCAPE_C_BUFFER_SIZE]; + + size_t i, j; + for (i = 0, j = 0; i < len; ++i) { + size_t rlen = EscapeC(str[i], (i + 1 < len ? str[i + 1] : 0), buffer); + + if (rlen > 1) { + output.Write(str + j, i - j); + j = i + 1; + output.Write(buffer, rlen); + } + } + + if (j > 0) { + output.Write(str + j, len - j); + } else { + output.Write(str, len); + } } - void EscapeC(const char* str, size_t len, IOutputStream& output) { - char buffer[ESCAPE_C_BUFFER_SIZE]; - - size_t i, j; - for (i = 0, j = 0; i < len; ++i) { - size_t rlen = EscapeC(str[i], (i + 1 < len ? str[i + 1] : 0), buffer); - - if (rlen > 1) { - output.Write(str + j, i - j); - j = i + 1; - output.Write(buffer, rlen); - } - } - - if (j > 0) { - output.Write(str + j, len - j); - } else { - output.Write(str, len); - } - } - - TString FloatToStringWithNanInf(double value) { - if (std::isfinite(value)) { - return ::ToString(value); - } + TString FloatToStringWithNanInf(double value) { + if (std::isfinite(value)) { + return ::ToString(value); + } static const TStringBuf nanLiteral = "%nan"; static const TStringBuf infLiteral = "%inf"; static const TStringBuf negativeInfLiteral = "%-inf"; - TStringBuf str; - if (std::isnan(value)) { - str = nanLiteral; - } else if (value > 0) { - str = infLiteral; - } else { - str = negativeInfLiteral; - } + TStringBuf str; + if (std::isnan(value)) { + str = nanLiteral; + } else if (value > 0) { + str = infLiteral; + } else { + str = negativeInfLiteral; + } return TString(str.data(), str.size()); - } - - } + } - //////////////////////////////////////////////////////////////////////////////// - - TYsonWriter::TYsonWriter( - IOutputStream* stream, - EYsonFormat format, - EYsonType type, - bool enableRaw) - : Stream(stream) - , Format(format) - , Type(type) - , EnableRaw(enableRaw) - , Depth(0) - , BeforeFirstItem(true) - { - Y_ASSERT(stream); } - void TYsonWriter::WriteIndent() { - for (int i = 0; i < IndentSize * Depth; ++i) { - Stream->Write(' '); - } - } - - bool TYsonWriter::IsTopLevelFragmentContext() const { + //////////////////////////////////////////////////////////////////////////////// + + TYsonWriter::TYsonWriter( + IOutputStream* stream, + EYsonFormat format, + EYsonType type, + bool enableRaw) + : Stream(stream) + , Format(format) + , Type(type) + , EnableRaw(enableRaw) + , Depth(0) + , BeforeFirstItem(true) + { + Y_ASSERT(stream); + } + + void TYsonWriter::WriteIndent() { + for (int i = 0; i < IndentSize * Depth; ++i) { + Stream->Write(' '); + } + } + + bool TYsonWriter::IsTopLevelFragmentContext() const { return Depth == 0 && (Type == ::NYson::EYsonType::ListFragment || Type == ::NYson::EYsonType::MapFragment); } - void TYsonWriter::EndNode() { - if (IsTopLevelFragmentContext()) { - ETokenType separatorToken = + void TYsonWriter::EndNode() { + if (IsTopLevelFragmentContext()) { + ETokenType separatorToken = Type == ::NYson::EYsonType::ListFragment - ? ListItemSeparatorToken - : KeyedItemSeparatorToken; - Stream->Write(TokenTypeToChar(separatorToken)); + ? ListItemSeparatorToken + : KeyedItemSeparatorToken; + Stream->Write(TokenTypeToChar(separatorToken)); if (Format == EYsonFormat::Text || Format == EYsonFormat::Pretty) { - Stream->Write('\n'); - } + Stream->Write('\n'); + } } } - void TYsonWriter::BeginCollection(ETokenType beginToken) { - Stream->Write(TokenTypeToChar(beginToken)); - ++Depth; - BeforeFirstItem = true; - } - - void TYsonWriter::CollectionItem(ETokenType separatorToken) { - if (!IsTopLevelFragmentContext()) { - if (!BeforeFirstItem) { - Stream->Write(TokenTypeToChar(separatorToken)); - } - + void TYsonWriter::BeginCollection(ETokenType beginToken) { + Stream->Write(TokenTypeToChar(beginToken)); + ++Depth; + BeforeFirstItem = true; + } + + void TYsonWriter::CollectionItem(ETokenType separatorToken) { + if (!IsTopLevelFragmentContext()) { + if (!BeforeFirstItem) { + Stream->Write(TokenTypeToChar(separatorToken)); + } + if (Format == EYsonFormat::Pretty) { - Stream->Write('\n'); - WriteIndent(); - } + Stream->Write('\n'); + WriteIndent(); + } } - BeforeFirstItem = false; - } - - void TYsonWriter::EndCollection(ETokenType endToken) { - --Depth; + BeforeFirstItem = false; + } + + void TYsonWriter::EndCollection(ETokenType endToken) { + --Depth; if (Format == EYsonFormat::Pretty && !BeforeFirstItem) { Stream->Write('\n'); WriteIndent(); } - Stream->Write(TokenTypeToChar(endToken)); - BeforeFirstItem = false; + Stream->Write(TokenTypeToChar(endToken)); + BeforeFirstItem = false; } - void TYsonWriter::WriteStringScalar(const TStringBuf& value) { + void TYsonWriter::WriteStringScalar(const TStringBuf& value) { if (Format == EYsonFormat::Binary) { - Stream->Write(NDetail::StringMarker); - WriteVarInt32(Stream, static_cast<i32>(value.length())); - Stream->Write(value.begin(), value.length()); - } else { - Stream->Write('"'); - EscapeC(value.data(), value.length(), *Stream); - Stream->Write('"'); - } - } + Stream->Write(NDetail::StringMarker); + WriteVarInt32(Stream, static_cast<i32>(value.length())); + Stream->Write(value.begin(), value.length()); + } else { + Stream->Write('"'); + EscapeC(value.data(), value.length(), *Stream); + Stream->Write('"'); + } + } void TYsonWriter::OnStringScalar(TStringBuf value) { - WriteStringScalar(value); - EndNode(); + WriteStringScalar(value); + EndNode(); } - void TYsonWriter::OnInt64Scalar(i64 value) { + void TYsonWriter::OnInt64Scalar(i64 value) { if (Format == EYsonFormat::Binary) { - Stream->Write(NDetail::Int64Marker); - WriteVarInt64(Stream, value); - } else { - Stream->Write(::ToString(value)); - } - EndNode(); + Stream->Write(NDetail::Int64Marker); + WriteVarInt64(Stream, value); + } else { + Stream->Write(::ToString(value)); + } + EndNode(); } - void TYsonWriter::OnUint64Scalar(ui64 value) { + void TYsonWriter::OnUint64Scalar(ui64 value) { if (Format == EYsonFormat::Binary) { - Stream->Write(NDetail::Uint64Marker); - WriteVarUInt64(Stream, value); - } else { - Stream->Write(::ToString(value)); - Stream->Write("u"); - } - EndNode(); + Stream->Write(NDetail::Uint64Marker); + WriteVarUInt64(Stream, value); + } else { + Stream->Write(::ToString(value)); + Stream->Write("u"); + } + EndNode(); } - void TYsonWriter::OnDoubleScalar(double value) { + void TYsonWriter::OnDoubleScalar(double value) { if (Format == EYsonFormat::Binary) { - Stream->Write(NDetail::DoubleMarker); - Stream->Write(&value, sizeof(double)); - } else { - auto str = FloatToStringWithNanInf(value); - Stream->Write(str); - if (str.find('.') == TString::npos && str.find('e') == TString::npos && std::isfinite(value)) { - Stream->Write("."); - } - } - EndNode(); + Stream->Write(NDetail::DoubleMarker); + Stream->Write(&value, sizeof(double)); + } else { + auto str = FloatToStringWithNanInf(value); + Stream->Write(str); + if (str.find('.') == TString::npos && str.find('e') == TString::npos && std::isfinite(value)) { + Stream->Write("."); + } + } + EndNode(); } - void TYsonWriter::OnBooleanScalar(bool value) { + void TYsonWriter::OnBooleanScalar(bool value) { if (Format == EYsonFormat::Binary) { - Stream->Write(value ? NDetail::TrueMarker : NDetail::FalseMarker); - } else { - Stream->Write(value ? "%true" : "%false"); + Stream->Write(value ? NDetail::TrueMarker : NDetail::FalseMarker); + } else { + Stream->Write(value ? "%true" : "%false"); } - EndNode(); + EndNode(); } - void TYsonWriter::OnEntity() { - Stream->Write(TokenTypeToChar(EntityToken)); - EndNode(); + void TYsonWriter::OnEntity() { + Stream->Write(TokenTypeToChar(EntityToken)); + EndNode(); } - void TYsonWriter::OnBeginList() { - BeginCollection(BeginListToken); - } + void TYsonWriter::OnBeginList() { + BeginCollection(BeginListToken); + } - void TYsonWriter::OnListItem() { - CollectionItem(ListItemSeparatorToken); - } + void TYsonWriter::OnListItem() { + CollectionItem(ListItemSeparatorToken); + } - void TYsonWriter::OnEndList() { - EndCollection(EndListToken); - EndNode(); - } + void TYsonWriter::OnEndList() { + EndCollection(EndListToken); + EndNode(); + } - void TYsonWriter::OnBeginMap() { - BeginCollection(BeginMapToken); - } + void TYsonWriter::OnBeginMap() { + BeginCollection(BeginMapToken); + } void TYsonWriter::OnKeyedItem(TStringBuf key) { - CollectionItem(KeyedItemSeparatorToken); + CollectionItem(KeyedItemSeparatorToken); - WriteStringScalar(key); + WriteStringScalar(key); if (Format == NYson::EYsonFormat::Pretty) { - Stream->Write(' '); - } - Stream->Write(TokenTypeToChar(KeyValueSeparatorToken)); + Stream->Write(' '); + } + Stream->Write(TokenTypeToChar(KeyValueSeparatorToken)); if (Format == NYson::EYsonFormat::Pretty) { - Stream->Write(' '); - } + Stream->Write(' '); + } - BeforeFirstItem = false; + BeforeFirstItem = false; } - - void TYsonWriter::OnEndMap() { - EndCollection(EndMapToken); - EndNode(); + + void TYsonWriter::OnEndMap() { + EndCollection(EndMapToken); + EndNode(); } - void TYsonWriter::OnBeginAttributes() { - BeginCollection(BeginAttributesToken); - } + void TYsonWriter::OnBeginAttributes() { + BeginCollection(BeginAttributesToken); + } - void TYsonWriter::OnEndAttributes() { - EndCollection(EndAttributesToken); + void TYsonWriter::OnEndAttributes() { + EndCollection(EndAttributesToken); if (Format == NYson::EYsonFormat::Pretty) { - Stream->Write(' '); - } - } + Stream->Write(' '); + } + } void TYsonWriter::OnRaw(TStringBuf yson, EYsonType type) { - if (EnableRaw) { - Stream->Write(yson); - BeforeFirstItem = false; - } else { - TYsonConsumerBase::OnRaw(yson, type); - } + if (EnableRaw) { + Stream->Write(yson); + BeforeFirstItem = false; + } else { + TYsonConsumerBase::OnRaw(yson, type); + } + } + + TYsonWriter::TState TYsonWriter::State() const { + TState state; + state.Depth = Depth; + state.BeforeFirstItem = BeforeFirstItem; + return state; } - TYsonWriter::TState TYsonWriter::State() const { - TState state; - state.Depth = Depth; - state.BeforeFirstItem = BeforeFirstItem; - return state; + void TYsonWriter::Reset(const TState& state) { + Depth = state.Depth; + BeforeFirstItem = state.BeforeFirstItem; } - void TYsonWriter::Reset(const TState& state) { - Depth = state.Depth; - BeforeFirstItem = state.BeforeFirstItem; - } - - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - void ReformatYsonStream( + void ReformatYsonStream( IInputStream* input, - IOutputStream* output, - EYsonFormat format, - EYsonType type) { - TYsonWriter writer(output, format, type); - TYsonParser parser(&writer, input, type); - parser.Parse(); - } - - //////////////////////////////////////////////////////////////////////////////// + IOutputStream* output, + EYsonFormat format, + EYsonType type) { + TYsonWriter writer(output, format, type); + TYsonParser parser(&writer, input, type); + parser.Parse(); + } + + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/writer.h b/library/cpp/yson/writer.h index 40f5d7d501..db4ed0f5e6 100644 --- a/library/cpp/yson/writer.h +++ b/library/cpp/yson/writer.h @@ -10,80 +10,80 @@ class IOutputStream; class IZeroCopyInput; namespace NYson { - //////////////////////////////////////////////////////////////////////////////// - - class TYsonWriter - : public TYsonConsumerBase, - private TNonCopyable { - public: - class TState { - private: - int Depth; - bool BeforeFirstItem; - - friend class TYsonWriter; - }; - - public: - TYsonWriter( - IOutputStream* stream, + //////////////////////////////////////////////////////////////////////////////// + + class TYsonWriter + : public TYsonConsumerBase, + private TNonCopyable { + public: + class TState { + private: + int Depth; + bool BeforeFirstItem; + + friend class TYsonWriter; + }; + + public: + TYsonWriter( + IOutputStream* stream, EYsonFormat format = EYsonFormat::Binary, EYsonType type = ::NYson::EYsonType::Node, - bool enableRaw = false); + bool enableRaw = false); void OnStringScalar(TStringBuf value) override; - void OnInt64Scalar(i64 value) override; - void OnUint64Scalar(ui64 value) override; - void OnDoubleScalar(double value) override; - void OnBooleanScalar(bool value) override; - void OnEntity() override; + void OnInt64Scalar(i64 value) override; + void OnUint64Scalar(ui64 value) override; + void OnDoubleScalar(double value) override; + void OnBooleanScalar(bool value) override; + void OnEntity() override; - void OnBeginList() override; - void OnListItem() override; - void OnEndList() override; + void OnBeginList() override; + void OnListItem() override; + void OnEndList() override; - void OnBeginMap() override; + void OnBeginMap() override; void OnKeyedItem(TStringBuf key) override; - void OnEndMap() override; + void OnEndMap() override; - void OnBeginAttributes() override; - void OnEndAttributes() override; + void OnBeginAttributes() override; + void OnEndAttributes() override; void OnRaw(TStringBuf yson, EYsonType type = ::NYson::EYsonType::Node) override; - TState State() const; - void Reset(const TState& state); + TState State() const; + void Reset(const TState& state); - protected: - IOutputStream* Stream; - EYsonFormat Format; - EYsonType Type; - bool EnableRaw; + protected: + IOutputStream* Stream; + EYsonFormat Format; + EYsonType Type; + bool EnableRaw; - int Depth; - bool BeforeFirstItem; + int Depth; + bool BeforeFirstItem; - static const int IndentSize = 4; + static const int IndentSize = 4; - void WriteIndent(); - void WriteStringScalar(const TStringBuf& value); + void WriteIndent(); + void WriteStringScalar(const TStringBuf& value); - void BeginCollection(ETokenType beginToken); - void CollectionItem(ETokenType separatorToken); - void EndCollection(ETokenType endToken); + void BeginCollection(ETokenType beginToken); + void CollectionItem(ETokenType separatorToken); + void EndCollection(ETokenType endToken); - bool IsTopLevelFragmentContext() const; - void EndNode(); - }; + bool IsTopLevelFragmentContext() const; + void EndNode(); + }; - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - void ReformatYsonStream( + void ReformatYsonStream( IInputStream* input, - IOutputStream* output, + IOutputStream* output, EYsonFormat format = EYsonFormat::Binary, EYsonType type = ::NYson::EYsonType::Node); - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// } // namespace NYson diff --git a/library/cpp/yson/zigzag.h b/library/cpp/yson/zigzag.h index 2f1190508f..e36df71714 100644 --- a/library/cpp/yson/zigzag.h +++ b/library/cpp/yson/zigzag.h @@ -3,29 +3,29 @@ #include <util/system/defaults.h> namespace NYson { - //////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////// - //! Functions that provide coding of integers with property: 0 <= f(x) <= 2 * |x| - //! Actually taken 'as is' from protobuf/wire_format_lite.h + //! Functions that provide coding of integers with property: 0 <= f(x) <= 2 * |x| + //! Actually taken 'as is' from protobuf/wire_format_lite.h - inline ui32 ZigZagEncode32(i32 n) { - // Note: the right-shift must be arithmetic - return (ui32(n) << 1) ^ (n >> 31); - } + inline ui32 ZigZagEncode32(i32 n) { + // Note: the right-shift must be arithmetic + return (ui32(n) << 1) ^ (n >> 31); + } - inline i32 ZigZagDecode32(ui32 n) { - return (n >> 1) ^ -static_cast<i32>(n & 1); - } + inline i32 ZigZagDecode32(ui32 n) { + return (n >> 1) ^ -static_cast<i32>(n & 1); + } - inline ui64 ZigZagEncode64(i64 n) { - // Note: the right-shift must be arithmetic - return (ui64(n) << 1) ^ (n >> 63); - } + inline ui64 ZigZagEncode64(i64 n) { + // Note: the right-shift must be arithmetic + return (ui64(n) << 1) ^ (n >> 63); + } - inline i64 ZigZagDecode64(ui64 n) { - return (n >> 1) ^ -static_cast<i64>(n & 1); - } - - //////////////////////////////////////////////////////////////////////////////// + inline i64 ZigZagDecode64(ui64 n) { + return (n >> 1) ^ -static_cast<i64>(n & 1); + } + //////////////////////////////////////////////////////////////////////////////// + } // namespace NYson diff --git a/library/cpp/yson_pull/bridge.h b/library/cpp/yson_pull/bridge.h index ac767dcba0..df03933a50 100644 --- a/library/cpp/yson_pull/bridge.h +++ b/library/cpp/yson_pull/bridge.h @@ -5,30 +5,30 @@ #include "writer.h" namespace NYsonPull { - //! \brief Connect YSON stream producer and consumer. - //! - //! Useful for writing YSON stream filters. - //! \p Producer must have a \p next_event() method (like \p NYsonPull::reader). - //! \p Consumer must be like \p NYsonPull::consumer interface. - template <typename Producer, typename Consumer> + //! \brief Connect YSON stream producer and consumer. + //! + //! Useful for writing YSON stream filters. + //! \p Producer must have a \p next_event() method (like \p NYsonPull::reader). + //! \p Consumer must be like \p NYsonPull::consumer interface. + template <typename Producer, typename Consumer> inline void Bridge(Producer&& producer, Consumer&& consumer) { - for (;;) { + for (;;) { auto& event = producer.NextEvent(); consumer.OnEvent(event); if (event.Type() == EEventType::EndStream) { - break; - } + break; + } } } - template <typename Producer> + template <typename Producer> inline void Bridge(Producer&& producer, TWriter& writer_) { Bridge(std::forward<Producer>(producer), writer_.GetConsumer()); - } + } - template <typename Producer> + template <typename Producer> inline void Bridge(Producer&& producer, TWriter&& writer_) { Bridge(std::forward<Producer>(producer), writer_.GetConsumer()); - } + } -} +} diff --git a/library/cpp/yson_pull/buffer.h b/library/cpp/yson_pull/buffer.h index 04c9220ef3..e434872c5e 100644 --- a/library/cpp/yson_pull/buffer.h +++ b/library/cpp/yson_pull/buffer.h @@ -6,74 +6,74 @@ #include <cstddef> namespace NYsonPull { - //! \brief A non-owning buffer model. - //! - //! Represents a \p pos pointer moving between \p begin and \p end. - template <typename T> - class buffer { - T* begin_ = nullptr; - T* pos_ = nullptr; - T* end_ = nullptr; + //! \brief A non-owning buffer model. + //! + //! Represents a \p pos pointer moving between \p begin and \p end. + template <typename T> + class buffer { + T* begin_ = nullptr; + T* pos_ = nullptr; + T* end_ = nullptr; - public: - T* begin() const noexcept { - return begin_; - } - T* pos() const noexcept { - return pos_; - } - T* end() const noexcept { - return end_; - } + public: + T* begin() const noexcept { + return begin_; + } + T* pos() const noexcept { + return pos_; + } + T* end() const noexcept { + return end_; + } - //! \brief Amount of data after current position. - size_t available() const noexcept { - return end_ - pos_; - } + //! \brief Amount of data after current position. + size_t available() const noexcept { + return end_ - pos_; + } - //! \brief Amount of data before current position. - size_t used() const noexcept { - return pos_ - begin_; - } + //! \brief Amount of data before current position. + size_t used() const noexcept { + return pos_ - begin_; + } - //! \brief Move current position \p nbytes forward. - void advance(size_t nbytes) noexcept { - Y_ASSERT(pos_ + nbytes <= end_); - pos_ += nbytes; - } + //! \brief Move current position \p nbytes forward. + void advance(size_t nbytes) noexcept { + Y_ASSERT(pos_ + nbytes <= end_); + pos_ += nbytes; + } - //! \brief Reset buffer pointers. - void reset(T* new_begin, T* new_end, T* new_pos) { - begin_ = new_begin; - pos_ = new_pos; - end_ = new_end; - } + //! \brief Reset buffer pointers. + void reset(T* new_begin, T* new_end, T* new_pos) { + begin_ = new_begin; + pos_ = new_pos; + end_ = new_end; + } - //! \brief Reset buffer to beginning - void reset(T* new_begin, T* new_end) { - reset(new_begin, new_end, new_begin); - } - }; + //! \brief Reset buffer to beginning + void reset(T* new_begin, T* new_end) { + reset(new_begin, new_end, new_begin); + } + }; - class output_buffer: public buffer<ui8> { - public: - //! \brief An output buffer is empty when there is no data written to it. - bool is_empty() const noexcept { - return pos() == begin(); - } + class output_buffer: public buffer<ui8> { + public: + //! \brief An output buffer is empty when there is no data written to it. + bool is_empty() const noexcept { + return pos() == begin(); + } - //! \brief An output buffer is full when there is no space to write more data to it. - bool is_full() const noexcept { - return pos() == end(); - } - }; + //! \brief An output buffer is full when there is no space to write more data to it. + bool is_full() const noexcept { + return pos() == end(); + } + }; - class input_buffer: public buffer<const ui8> { - public: - //! An input stream is empty when there is no data to read in it. - bool is_empty() const noexcept { - return pos() == end(); - } - }; + class input_buffer: public buffer<const ui8> { + public: + //! An input stream is empty when there is no data to read in it. + bool is_empty() const noexcept { + return pos() == end(); + } + }; -} +} diff --git a/library/cpp/yson_pull/consumer.cpp b/library/cpp/yson_pull/consumer.cpp index c238e0a6fb..e3bce1b29d 100644 --- a/library/cpp/yson_pull/consumer.cpp +++ b/library/cpp/yson_pull/consumer.cpp @@ -30,7 +30,7 @@ void IConsumer::OnScalar(const TScalar& value) { OnScalarString(value.AsString()); break; - default: + default: Y_UNREACHABLE(); } } @@ -77,7 +77,7 @@ void IConsumer::OnEvent(const TEvent& value) { OnScalar(value.AsScalar()); break; - default: + default: Y_UNREACHABLE(); } } diff --git a/library/cpp/yson_pull/consumer.h b/library/cpp/yson_pull/consumer.h index f3b1398d4c..64d255a0cb 100644 --- a/library/cpp/yson_pull/consumer.h +++ b/library/cpp/yson_pull/consumer.h @@ -6,9 +6,9 @@ #include <util/system/types.h> namespace NYsonPull { - class IConsumer { - public: - virtual ~IConsumer() = default; + class IConsumer { + public: + virtual ~IConsumer() = default; virtual void OnBeginStream() = 0; virtual void OnEndStream() = 0; @@ -33,5 +33,5 @@ namespace NYsonPull { virtual void OnScalar(const TScalar& value); virtual void OnEvent(const TEvent& value); - }; -} + }; +} diff --git a/library/cpp/yson_pull/detail/byte_reader.h b/library/cpp/yson_pull/detail/byte_reader.h index 7cea50d323..915277518a 100644 --- a/library/cpp/yson_pull/detail/byte_reader.h +++ b/library/cpp/yson_pull/detail/byte_reader.h @@ -6,69 +6,69 @@ #include <library/cpp/yson_pull/input.h> -namespace NYsonPull { - namespace NDetail { - template <class StreamCounter> - class byte_reader { - NYsonPull::NInput::IStream& stream_; - StreamCounter stream_counter_; +namespace NYsonPull { + namespace NDetail { + template <class StreamCounter> + class byte_reader { + NYsonPull::NInput::IStream& stream_; + StreamCounter stream_counter_; - public: - byte_reader(NYsonPull::NInput::IStream& stream) - : stream_(stream) - { - } + public: + byte_reader(NYsonPull::NInput::IStream& stream) + : stream_(stream) + { + } - // const-ness added to prevent direct stream mutation - const NYsonPull::NInput::IStream& stream() { - return stream_; - } + // const-ness added to prevent direct stream mutation + const NYsonPull::NInput::IStream& stream() { + return stream_; + } - template <typename... Args> - ATTRIBUTE(noinline, cold) - void fail[[noreturn]](const char* msg, Args&&... args) { - NYsonPull::NDetail::fail( - stream_counter_.info(), - msg, - std::forward<Args>(args)...); - } + template <typename... Args> + ATTRIBUTE(noinline, cold) + void fail[[noreturn]](const char* msg, Args&&... args) { + NYsonPull::NDetail::fail( + stream_counter_.info(), + msg, + std::forward<Args>(args)...); + } - template <bool AllowFinish> - void fill_buffer() { - stream_.fill_buffer(); + template <bool AllowFinish> + void fill_buffer() { + stream_.fill_buffer(); - if (!AllowFinish) { - auto& buf = stream_.buffer(); + if (!AllowFinish) { + auto& buf = stream_.buffer(); if (Y_UNLIKELY(buf.is_empty() && stream_.at_end())) { - fail("Premature end of stream"); - } - } - } + fail("Premature end of stream"); + } + } + } - void fill_buffer() { - return fill_buffer<true>(); + void fill_buffer() { + return fill_buffer<true>(); } - template <bool AllowFinish> - ui8 get_byte() { - fill_buffer<AllowFinish>(); - auto& buf = stream_.buffer(); - return !buf.is_empty() - ? *buf.pos() - : ui8{'\0'}; - } + template <bool AllowFinish> + ui8 get_byte() { + fill_buffer<AllowFinish>(); + auto& buf = stream_.buffer(); + return !buf.is_empty() + ? *buf.pos() + : ui8{'\0'}; + } - ui8 get_byte() { - return get_byte<true>(); - } + ui8 get_byte() { + return get_byte<true>(); + } - void advance(size_t bytes) { - auto& buf = stream_.buffer(); - stream_counter_.update( - buf.pos(), - buf.pos() + bytes); - buf.advance(bytes); - } - }; + void advance(size_t bytes) { + auto& buf = stream_.buffer(); + stream_counter_.update( + buf.pos(), + buf.pos() + bytes); + buf.advance(bytes); + } + }; } -} +} diff --git a/library/cpp/yson_pull/detail/byte_writer.h b/library/cpp/yson_pull/detail/byte_writer.h index dc1d4b4b96..9a755912d7 100644 --- a/library/cpp/yson_pull/detail/byte_writer.h +++ b/library/cpp/yson_pull/detail/byte_writer.h @@ -9,69 +9,69 @@ #include <cstddef> #include <cstring> -namespace NYsonPull { - namespace NDetail { - template <class StreamCounter> - class byte_writer { - NYsonPull::NOutput::IStream& stream_; - StreamCounter stream_counter_; +namespace NYsonPull { + namespace NDetail { + template <class StreamCounter> + class byte_writer { + NYsonPull::NOutput::IStream& stream_; + StreamCounter stream_counter_; - public: - byte_writer(NYsonPull::NOutput::IStream& stream) - : stream_(stream) - { - } + public: + byte_writer(NYsonPull::NOutput::IStream& stream) + : stream_(stream) + { + } - // const-ness added to prevent direct stream mutation - const NYsonPull::NOutput::IStream& stream() { - return stream_; - } - const StreamCounter& counter() { - return stream_counter_; - } + // const-ness added to prevent direct stream mutation + const NYsonPull::NOutput::IStream& stream() { + return stream_; + } + const StreamCounter& counter() { + return stream_counter_; + } - void flush_buffer() { - stream_.flush_buffer(); - } + void flush_buffer() { + stream_.flush_buffer(); + } - void advance(size_t bytes) { - auto& buf = stream_.buffer(); - stream_counter_.update( - buf.pos(), - buf.pos() + bytes); - buf.advance(bytes); - } + void advance(size_t bytes) { + auto& buf = stream_.buffer(); + stream_counter_.update( + buf.pos(), + buf.pos() + bytes); + buf.advance(bytes); + } - void write(ui8 c) { - auto& buf = stream_.buffer(); + void write(ui8 c) { + auto& buf = stream_.buffer(); if (Y_LIKELY(!buf.is_full())) { - *buf.pos() = c; - advance(1); - } else { - auto ptr = reinterpret_cast<char*>(&c); - stream_counter_.update(&c, &c + 1); - stream_.flush_buffer({ptr, 1}); - } - } + *buf.pos() = c; + advance(1); + } else { + auto ptr = reinterpret_cast<char*>(&c); + stream_counter_.update(&c, &c + 1); + stream_.flush_buffer({ptr, 1}); + } + } - void write(const ui8* data, size_t size) { - auto& buf = stream_.buffer(); - auto free_buf = buf.available(); + void write(const ui8* data, size_t size) { + auto& buf = stream_.buffer(); + auto free_buf = buf.available(); if (Y_LIKELY(size < free_buf)) { - ::memcpy(buf.pos(), data, size); - advance(size); - } else { - if (!buf.is_full()) { - ::memcpy(buf.pos(), data, free_buf); - advance(free_buf); - data += free_buf; - size -= free_buf; - } - stream_counter_.update(data, data + size); - stream_.flush_buffer({reinterpret_cast<const char*>(data), - size}); - } + ::memcpy(buf.pos(), data, size); + advance(size); + } else { + if (!buf.is_full()) { + ::memcpy(buf.pos(), data, free_buf); + advance(free_buf); + data += free_buf; + size -= free_buf; + } + stream_counter_.update(data, data + size); + stream_.flush_buffer({reinterpret_cast<const char*>(data), + size}); + } } - }; + }; } -} +} diff --git a/library/cpp/yson_pull/detail/cescape.h b/library/cpp/yson_pull/detail/cescape.h index 1ea150e69a..d12936e63c 100644 --- a/library/cpp/yson_pull/detail/cescape.h +++ b/library/cpp/yson_pull/detail/cescape.h @@ -35,109 +35,109 @@ * */ -namespace NYsonPull { - namespace NDetail { - namespace NCEscape { +namespace NYsonPull { + namespace NDetail { + namespace NCEscape { inline void encode(TString& dest, TStringBuf data) { - NImpl::escape_impl( - reinterpret_cast<const ui8*>(data.data()), - data.size(), - [&](const ui8* str, size_t size) { - dest.append( - reinterpret_cast<const char*>(str), - size); - }); - } + NImpl::escape_impl( + reinterpret_cast<const ui8*>(data.data()), + data.size(), + [&](const ui8* str, size_t size) { + dest.append( + reinterpret_cast<const char*>(str), + size); + }); + } - // dest must have at least 4*data.size() bytes available - inline size_t encode(ui8* dest, TStringBuf data) { - auto* dest_begin = dest; - NImpl::escape_impl( - reinterpret_cast<const ui8*>(data.data()), - data.size(), - [&](const ui8* str, size_t size) { - ::memcpy(dest, str, size); - dest += size; - }); - return dest - dest_begin; - } + // dest must have at least 4*data.size() bytes available + inline size_t encode(ui8* dest, TStringBuf data) { + auto* dest_begin = dest; + NImpl::escape_impl( + reinterpret_cast<const ui8*>(data.data()), + data.size(), + [&](const ui8* str, size_t size) { + ::memcpy(dest, str, size); + dest += size; + }); + return dest - dest_begin; + } - template <typename U> - void encode(byte_writer<U>& dest, TStringBuf data) { - auto& buffer = dest.stream().buffer(); + template <typename U> + void encode(byte_writer<U>& dest, TStringBuf data) { + auto& buffer = dest.stream().buffer(); if (Y_LIKELY(buffer.available() >= data.size() * 4)) { - auto size = encode(buffer.pos(), data); - dest.advance(size); - } else { - NImpl::escape_impl( - reinterpret_cast<const ui8*>(data.data()), - data.size(), - [&](const ui8* str, size_t size) { - dest.write(str, size); - }); - } - } + auto size = encode(buffer.pos(), data); + dest.advance(size); + } else { + NImpl::escape_impl( + reinterpret_cast<const ui8*>(data.data()), + data.size(), + [&](const ui8* str, size_t size) { + dest.write(str, size); + }); + } + } inline TString encode(TStringBuf data) { TString result; - result.reserve(data.size()); - encode(result, data); - return result; + result.reserve(data.size()); + encode(result, data); + return result; } inline void decode(TString& dest, TStringBuf data) { - NImpl::unescape_impl( - reinterpret_cast<const ui8*>(data.begin()), - reinterpret_cast<const ui8*>(data.end()), - [&](ui8 c) { - dest += c; - }, - [&](const ui8* p, size_t len) { - dest.append(reinterpret_cast<const char*>(p), len); - }); - } + NImpl::unescape_impl( + reinterpret_cast<const ui8*>(data.begin()), + reinterpret_cast<const ui8*>(data.end()), + [&](ui8 c) { + dest += c; + }, + [&](const ui8* p, size_t len) { + dest.append(reinterpret_cast<const char*>(p), len); + }); + } inline void decode_inplace(TVector<ui8>& data) { - auto* out = static_cast<ui8*>( - ::memchr(data.data(), '\\', data.size())); - if (out == nullptr) { - return; - } - NImpl::unescape_impl( - out, - data.data() + data.size(), - [&](ui8 c) { - *out++ = c; - }, - [&](const ui8* p, size_t len) { - ::memmove(out, p, len); - out += len; - }); - data.resize(out - &data[0]); - } + auto* out = static_cast<ui8*>( + ::memchr(data.data(), '\\', data.size())); + if (out == nullptr) { + return; + } + NImpl::unescape_impl( + out, + data.data() + data.size(), + [&](ui8 c) { + *out++ = c; + }, + [&](const ui8* p, size_t len) { + ::memmove(out, p, len); + out += len; + }); + data.resize(out - &data[0]); + } inline TString decode(TStringBuf data) { TString result; - result.reserve(data.size()); - decode(result, data); - return result; - } + result.reserve(data.size()); + decode(result, data); + return result; + } - ATTRIBUTE(noinline, cold) + ATTRIBUTE(noinline, cold) inline TString quote(TStringBuf str) { TString result; - result.reserve(str.size() + 16); - result += '"'; - encode(result, str); - result += '"'; - return result; - } + result.reserve(str.size() + 16); + result += '"'; + encode(result, str); + result += '"'; + return result; + } - ATTRIBUTE(noinline, cold) + ATTRIBUTE(noinline, cold) inline TString quote(ui8 ch) { - char c = ch; - return quote(TStringBuf(&c, 1)); - } - } - } // namespace NDetail -} + char c = ch; + return quote(TStringBuf(&c, 1)); + } + } + } // namespace NDetail +} diff --git a/library/cpp/yson_pull/detail/cescape_decode.h b/library/cpp/yson_pull/detail/cescape_decode.h index 2ee5dd9500..8e10cf3892 100644 --- a/library/cpp/yson_pull/detail/cescape_decode.h +++ b/library/cpp/yson_pull/detail/cescape_decode.h @@ -5,150 +5,150 @@ #include <algorithm> #include <cstring> -namespace NYsonPull { - namespace NDetail { - namespace NCEscape { - namespace NImpl { - inline ui8 as_digit(ui8 c) { - return c - ui8{'0'}; - } +namespace NYsonPull { + namespace NDetail { + namespace NCEscape { + namespace NImpl { + inline ui8 as_digit(ui8 c) { + return c - ui8{'0'}; + } - inline ui8 as_hexdigit(ui8 c) { - static constexpr ui8 hex_decode_map[256] = { - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, - 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255}; + inline ui8 as_hexdigit(ui8 c) { + static constexpr ui8 hex_decode_map[256] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255}; - return hex_decode_map[c]; - } + return hex_decode_map[c]; + } - inline const ui8* read_oct(ui8& result, const ui8* p, ui8 n) { - auto digit = ui8{0}; - while (n-- && (digit = as_digit(*p)) < 8) { - result = result * 8 + digit; - ++p; - } - return p; - } + inline const ui8* read_oct(ui8& result, const ui8* p, ui8 n) { + auto digit = ui8{0}; + while (n-- && (digit = as_digit(*p)) < 8) { + result = result * 8 + digit; + ++p; + } + return p; + } - inline const ui8* read_hex(ui8& result, const ui8* p, ui8 n) { - auto digit = ui8{0}; - while (n-- && (digit = as_hexdigit(*p)) < 16) { - result = result * 16 + digit; - ++p; - } - return p; - } + inline const ui8* read_hex(ui8& result, const ui8* p, ui8 n) { + auto digit = ui8{0}; + while (n-- && (digit = as_hexdigit(*p)) < 16) { + result = result * 16 + digit; + ++p; + } + return p; + } - inline const ui8* unescape_char_and_advance( - ui8& result, - const ui8* p, - const ui8* end) { - switch (*p) { - default: - result = *p; - ++p; - break; - case 'b': - result = '\b'; - ++p; - break; - case 'f': - result = '\f'; - ++p; - break; - case 'n': - result = '\n'; - ++p; - break; - case 'r': - result = '\r'; - ++p; - break; - case 't': - result = '\t'; - ++p; - break; + inline const ui8* unescape_char_and_advance( + ui8& result, + const ui8* p, + const ui8* end) { + switch (*p) { + default: + result = *p; + ++p; + break; + case 'b': + result = '\b'; + ++p; + break; + case 'f': + result = '\f'; + ++p; + break; + case 'n': + result = '\n'; + ++p; + break; + case 'r': + result = '\r'; + ++p; + break; + case 't': + result = '\t'; + ++p; + break; - case 'x': { - ++p; - result = 0; - auto* next = read_hex( - result, - p, std::min<ptrdiff_t>(2, end - p)); - if (next > p) { - p = next; - } else { - result = 'x'; - } - } break; + case 'x': { + ++p; + result = 0; + auto* next = read_hex( + result, + p, std::min<ptrdiff_t>(2, end - p)); + if (next > p) { + p = next; + } else { + result = 'x'; + } + } break; - case '0': - case '1': - case '2': - case '3': - result = 0; - p = read_oct( - result, - p, std::min<ptrdiff_t>(3, end - p)); - break; + case '0': + case '1': + case '2': + case '3': + result = 0; + p = read_oct( + result, + p, std::min<ptrdiff_t>(3, end - p)); + break; - case '4': - case '5': - case '6': - case '7': - result = 0; - p = read_oct( - result, - p, std::min<ptrdiff_t>(2, end - p)); - break; - } - return p; - } + case '4': + case '5': + case '6': + case '7': + result = 0; + p = read_oct( + result, + p, std::min<ptrdiff_t>(2, end - p)); + break; + } + return p; + } - template <typename T, typename U> - inline void unescape_impl( - const ui8* p, - const ui8* end, - T&& consume_one, - U&& consume_span) { - while (p < end) { - auto* escaped = static_cast<const ui8*>( - ::memchr(p, '\\', end - p)); - if (escaped == nullptr) { - consume_span(p, end - p); - return; - } else { - consume_span(p, escaped - p); - auto c = ui8{'\\'}; - p = escaped + 1; - if (p < end) { - p = unescape_char_and_advance(c, p, end); - } - consume_one(c); - } - } - } + template <typename T, typename U> + inline void unescape_impl( + const ui8* p, + const ui8* end, + T&& consume_one, + U&& consume_span) { + while (p < end) { + auto* escaped = static_cast<const ui8*>( + ::memchr(p, '\\', end - p)); + if (escaped == nullptr) { + consume_span(p, end - p); + return; + } else { + consume_span(p, escaped - p); + auto c = ui8{'\\'}; + p = escaped + 1; + if (p < end) { + p = unescape_char_and_advance(c, p, end); + } + consume_one(c); + } + } + } } - } // namespace NCEscape - } // namespace NDetail -} + } // namespace NCEscape + } // namespace NDetail +} diff --git a/library/cpp/yson_pull/detail/cescape_encode.h b/library/cpp/yson_pull/detail/cescape_encode.h index bf5765f1d9..d1dae0f96e 100644 --- a/library/cpp/yson_pull/detail/cescape_encode.h +++ b/library/cpp/yson_pull/detail/cescape_encode.h @@ -6,109 +6,109 @@ // Turns UTF-8 strings into unreadable garbage for no known reason //#define CESCAPE_STRICT_ASCII -namespace NYsonPull { - namespace NDetail { - namespace NCEscape { - namespace NImpl { - inline ui8 hex_digit(ui8 value) { - constexpr ui8 hex_digits[] = "0123456789ABCDEF"; - return hex_digits[value]; - } +namespace NYsonPull { + namespace NDetail { + namespace NCEscape { + namespace NImpl { + inline ui8 hex_digit(ui8 value) { + constexpr ui8 hex_digits[] = "0123456789ABCDEF"; + return hex_digits[value]; + } - inline ui8 oct_digit(ui8 value) { - return '0' + value; - } + inline ui8 oct_digit(ui8 value) { + return '0' + value; + } - inline bool is_printable(ui8 c) { + inline bool is_printable(ui8 c) { #ifdef CESCAPE_STRICT_ASCII - return c >= 32 && c <= 126; + return c >= 32 && c <= 126; #else - return c >= 32; + return c >= 32; #endif - } + } - inline bool is_hex_digit(ui8 c) { - return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); - } + inline bool is_hex_digit(ui8 c) { + return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f'); + } - inline bool is_oct_digit(ui8 c) { - return c >= '0' && c <= '7'; - } + inline bool is_oct_digit(ui8 c) { + return c >= '0' && c <= '7'; + } - constexpr size_t ESCAPE_C_BUFFER_SIZE = 4; + constexpr size_t ESCAPE_C_BUFFER_SIZE = 4; - inline size_t escape_char( - ui8 c, - ui8 next, - ui8 r[ESCAPE_C_BUFFER_SIZE]) { - // (1) Printable characters go as-is, except backslash and double quote. - // (2) Characters \r, \n, \t and \0 ... \7 replaced by their simple escape characters (if possible). - // (3) Otherwise, character is encoded using hexadecimal escape sequence (if possible), or octal. - if (c == '\"') { - r[0] = '\\'; - r[1] = '\"'; - return 2; - } else if (c == '\\') { - r[0] = '\\'; - r[1] = '\\'; - return 2; - } else if (is_printable(c)) { - r[0] = c; - return 1; - } else if (c == '\r') { - r[0] = '\\'; - r[1] = 'r'; - return 2; - } else if (c == '\n') { - r[0] = '\\'; - r[1] = 'n'; - return 2; - } else if (c == '\t') { - r[0] = '\\'; - r[1] = 't'; - return 2; - } else if (c < 8 && !is_oct_digit(next)) { - r[0] = '\\'; - r[1] = oct_digit(c); - return 2; - } else if (!is_hex_digit(next)) { - r[0] = '\\'; - r[1] = 'x'; - r[2] = hex_digit((c & 0xF0) >> 4); - r[3] = hex_digit((c & 0x0F) >> 0); - return 4; - } else { - r[0] = '\\'; - r[1] = oct_digit((c & 0700) >> 6); - r[2] = oct_digit((c & 0070) >> 3); - r[3] = oct_digit((c & 0007) >> 0); - return 4; - } - } + inline size_t escape_char( + ui8 c, + ui8 next, + ui8 r[ESCAPE_C_BUFFER_SIZE]) { + // (1) Printable characters go as-is, except backslash and double quote. + // (2) Characters \r, \n, \t and \0 ... \7 replaced by their simple escape characters (if possible). + // (3) Otherwise, character is encoded using hexadecimal escape sequence (if possible), or octal. + if (c == '\"') { + r[0] = '\\'; + r[1] = '\"'; + return 2; + } else if (c == '\\') { + r[0] = '\\'; + r[1] = '\\'; + return 2; + } else if (is_printable(c)) { + r[0] = c; + return 1; + } else if (c == '\r') { + r[0] = '\\'; + r[1] = 'r'; + return 2; + } else if (c == '\n') { + r[0] = '\\'; + r[1] = 'n'; + return 2; + } else if (c == '\t') { + r[0] = '\\'; + r[1] = 't'; + return 2; + } else if (c < 8 && !is_oct_digit(next)) { + r[0] = '\\'; + r[1] = oct_digit(c); + return 2; + } else if (!is_hex_digit(next)) { + r[0] = '\\'; + r[1] = 'x'; + r[2] = hex_digit((c & 0xF0) >> 4); + r[3] = hex_digit((c & 0x0F) >> 0); + return 4; + } else { + r[0] = '\\'; + r[1] = oct_digit((c & 0700) >> 6); + r[2] = oct_digit((c & 0070) >> 3); + r[3] = oct_digit((c & 0007) >> 0); + return 4; + } + } - template <typename T> - inline void escape_impl(const ui8* str, size_t len, T&& consume) { - ui8 buffer[ESCAPE_C_BUFFER_SIZE]; + template <typename T> + inline void escape_impl(const ui8* str, size_t len, T&& consume) { + ui8 buffer[ESCAPE_C_BUFFER_SIZE]; - size_t i, j; - for (i = 0, j = 0; i < len; ++i) { - auto next_char = i + 1 < len ? str[i + 1] : 0; - size_t rlen = escape_char(str[i], next_char, buffer); + size_t i, j; + for (i = 0, j = 0; i < len; ++i) { + auto next_char = i + 1 < len ? str[i + 1] : 0; + size_t rlen = escape_char(str[i], next_char, buffer); - if (rlen > 1) { - consume(str + j, i - j); - j = i + 1; - consume(buffer, rlen); - } - } - - if (j > 0) { - consume(str + j, len - j); - } else { - consume(str, len); - } - } - } - } // namespace NCEscape - } // namespace NDetail -} + if (rlen > 1) { + consume(str + j, i - j); + j = i + 1; + consume(buffer, rlen); + } + } + + if (j > 0) { + consume(str + j, len - j); + } else { + consume(str, len); + } + } + } + } // namespace NCEscape + } // namespace NDetail +} diff --git a/library/cpp/yson_pull/detail/fail.h b/library/cpp/yson_pull/detail/fail.h index 6937612d0b..ad869af510 100644 --- a/library/cpp/yson_pull/detail/fail.h +++ b/library/cpp/yson_pull/detail/fail.h @@ -6,15 +6,15 @@ #include <library/cpp/yson_pull/exceptions.h> #include <library/cpp/yson_pull/position_info.h> -namespace NYsonPull { - namespace NDetail { - template <typename... Args> - ATTRIBUTE(noreturn, noinline, cold) - void fail( - const TPositionInfo& info, - Args&&... args) { - auto formatted_message = format_string(std::forward<Args>(args)...); - throw NException::TBadInput(formatted_message, info); - } - } -} +namespace NYsonPull { + namespace NDetail { + template <typename... Args> + ATTRIBUTE(noreturn, noinline, cold) + void fail( + const TPositionInfo& info, + Args&&... args) { + auto formatted_message = format_string(std::forward<Args>(args)...); + throw NException::TBadInput(formatted_message, info); + } + } +} diff --git a/library/cpp/yson_pull/detail/format_string.h b/library/cpp/yson_pull/detail/format_string.h index 683fd1bf36..437dee23e4 100644 --- a/library/cpp/yson_pull/detail/format_string.h +++ b/library/cpp/yson_pull/detail/format_string.h @@ -4,23 +4,23 @@ #include <util/generic/string.h> #include <util/string/builder.h> -namespace NYsonPull { - namespace NDetail { - namespace NImpl { - inline void apply_args(TStringBuilder&) { - } +namespace NYsonPull { + namespace NDetail { + namespace NImpl { + inline void apply_args(TStringBuilder&) { + } - template <typename T, typename... Args> - inline void apply_args(TStringBuilder& builder, T&& arg, Args&&... args) { - apply_args(builder << arg, std::forward<Args>(args)...); - } - } + template <typename T, typename... Args> + inline void apply_args(TStringBuilder& builder, T&& arg, Args&&... args) { + apply_args(builder << arg, std::forward<Args>(args)...); + } + } - template <typename... Args> + template <typename... Args> TString format_string(Args&&... args) { - TStringBuilder builder; - NImpl::apply_args(builder, std::forward<Args>(args)...); + TStringBuilder builder; + NImpl::apply_args(builder, std::forward<Args>(args)...); return TString(std::move(builder)); - } + } } -} +} diff --git a/library/cpp/yson_pull/detail/input/buffered.h b/library/cpp/yson_pull/detail/input/buffered.h index 9b1482577f..a0e53cec04 100644 --- a/library/cpp/yson_pull/detail/input/buffered.h +++ b/library/cpp/yson_pull/detail/input/buffered.h @@ -8,28 +8,28 @@ #include <cstdio> #include <memory> -namespace NYsonPull { - namespace NDetail { - namespace NInput { - class TBuffered: public NYsonPull::NInput::IStream { - TArrayHolder<ui8> buffer_; - size_t size_; +namespace NYsonPull { + namespace NDetail { + namespace NInput { + class TBuffered: public NYsonPull::NInput::IStream { + TArrayHolder<ui8> buffer_; + size_t size_; - public: - explicit TBuffered(size_t buffer_size) - : buffer_{new ui8[buffer_size]} - , size_{buffer_size} { - } + public: + explicit TBuffered(size_t buffer_size) + : buffer_{new ui8[buffer_size]} + , size_{buffer_size} { + } - protected: - ui8* buffer_data() const { - return buffer_.Get(); - } + protected: + ui8* buffer_data() const { + return buffer_.Get(); + } - size_t buffer_size() const { - return size_; - } - }; - } - } // namespace NDetail -} + size_t buffer_size() const { + return size_; + } + }; + } + } // namespace NDetail +} diff --git a/library/cpp/yson_pull/detail/input/stdio_file.h b/library/cpp/yson_pull/detail/input/stdio_file.h index c412b7e59b..c27b736aa3 100644 --- a/library/cpp/yson_pull/detail/input/stdio_file.h +++ b/library/cpp/yson_pull/detail/input/stdio_file.h @@ -10,33 +10,33 @@ #include <cstdio> #include <memory> -namespace NYsonPull { - namespace NDetail { - namespace NInput { - class TStdioFile: public TBuffered { - FILE* file_; +namespace NYsonPull { + namespace NDetail { + namespace NInput { + class TStdioFile: public TBuffered { + FILE* file_; - public: - TStdioFile(FILE* file, size_t buffer_size) - : TBuffered(buffer_size) - , file_{file} { - } + public: + TStdioFile(FILE* file, size_t buffer_size) + : TBuffered(buffer_size) + , file_{file} { + } - protected: - result do_fill_buffer() override { - auto nread = ::fread(buffer_data(), 1, buffer_size(), file_); + protected: + result do_fill_buffer() override { + auto nread = ::fread(buffer_data(), 1, buffer_size(), file_); if (Y_UNLIKELY(nread == 0)) { if (ferror(file_)) { - throw NException::TSystemError(); - } + throw NException::TSystemError(); + } if (feof(file_)) { - return result::at_end; - } - } - buffer().reset(buffer_data(), buffer_data() + nread); - return result::have_more_data; - } - }; + return result::at_end; + } + } + buffer().reset(buffer_data(), buffer_data() + nread); + return result::have_more_data; + } + }; } - } // namespace NDetail -} + } // namespace NDetail +} diff --git a/library/cpp/yson_pull/detail/input/stream.h b/library/cpp/yson_pull/detail/input/stream.h index 791cd5a3f5..d93ac8eaf8 100644 --- a/library/cpp/yson_pull/detail/input/stream.h +++ b/library/cpp/yson_pull/detail/input/stream.h @@ -40,12 +40,12 @@ namespace NYsonPull { } }; - template <typename TBuffered> + template <typename TBuffered> class TOwned: public TStreamBase { TBuffered Input; public: - template <typename... Args> + template <typename... Args> explicit TOwned(Args&&... args) : Input(std::forward<Args>(args)...) { @@ -65,5 +65,5 @@ namespace NYsonPull { } }; } - } // namespace NDetail -} + } // namespace NDetail +} diff --git a/library/cpp/yson_pull/detail/lexer_base.h b/library/cpp/yson_pull/detail/lexer_base.h index 572bdb3d18..54454f8c6f 100644 --- a/library/cpp/yson_pull/detail/lexer_base.h +++ b/library/cpp/yson_pull/detail/lexer_base.h @@ -12,209 +12,209 @@ #include <util/generic/vector.h> #include <util/string/cast.h> -namespace NYsonPull { - namespace NDetail { - template <bool EnableLinePositionInfo> - class lexer_base: public byte_reader<stream_counter<EnableLinePositionInfo>> { - using Base = byte_reader< - stream_counter<EnableLinePositionInfo>>; +namespace NYsonPull { + namespace NDetail { + template <bool EnableLinePositionInfo> + class lexer_base: public byte_reader<stream_counter<EnableLinePositionInfo>> { + using Base = byte_reader< + stream_counter<EnableLinePositionInfo>>; TVector<ui8> token_buffer_; - TMaybe<size_t> memory_limit_; - - public: - lexer_base( - NYsonPull::NInput::IStream& buffer, - TMaybe<size_t> memory_limit) - : Base(buffer) - , memory_limit_{memory_limit} { - } - - ATTRIBUTE(noinline, hot) - ui8 skip_space_and_get_byte() { - auto& buf = Base::stream().buffer(); + TMaybe<size_t> memory_limit_; + + public: + lexer_base( + NYsonPull::NInput::IStream& buffer, + TMaybe<size_t> memory_limit) + : Base(buffer) + , memory_limit_{memory_limit} { + } + + ATTRIBUTE(noinline, hot) + ui8 skip_space_and_get_byte() { + auto& buf = Base::stream().buffer(); if (Y_LIKELY(!buf.is_empty())) { - auto ch = *buf.pos(); + auto ch = *buf.pos(); if (Y_LIKELY(!is_space(ch))) { - return ch; - } - } - return skip_space_and_get_byte_fallback(); - } - - ATTRIBUTE(hot) - ui8 get_byte() { - auto& buf = Base::stream().buffer(); + return ch; + } + } + return skip_space_and_get_byte_fallback(); + } + + ATTRIBUTE(hot) + ui8 get_byte() { + auto& buf = Base::stream().buffer(); if (Y_LIKELY(!buf.is_empty())) { - return *buf.pos(); - } - return Base::get_byte(); + return *buf.pos(); + } + return Base::get_byte(); } - number read_numeric() { - token_buffer_.clear(); - auto type = number_type::int64; - while (true) { + number read_numeric() { + token_buffer_.clear(); + auto type = number_type::int64; + while (true) { auto ch = this->Base::template get_byte<true>(); - if (isdigit(ch) || ch == '+' || ch == '-') { - token_buffer_.push_back(ch); - } else if (ch == '.' || ch == 'e' || ch == 'E') { - token_buffer_.push_back(ch); - type = number_type::float64; - } else if (ch == 'u') { - token_buffer_.push_back(ch); - type = number_type::uint64; + if (isdigit(ch) || ch == '+' || ch == '-') { + token_buffer_.push_back(ch); + } else if (ch == '.' || ch == 'e' || ch == 'E') { + token_buffer_.push_back(ch); + type = number_type::float64; + } else if (ch == 'u') { + token_buffer_.push_back(ch); + type = number_type::uint64; } else if (Y_UNLIKELY(isalpha(ch))) { - COLD_BLOCK_BYVALUE - Base::fail("Unexpected ", NCEscape::quote(ch), " in numeric literal"); - COLD_BLOCK_END - } else { - break; - } - check_memory_limit(); - Base::advance(1); - } - - auto str = token_buffer(); - try { - switch (type) { - case number_type::float64: - return FromString<double>(str); - case number_type::int64: - return FromString<i64>(str); - case number_type::uint64: - str.Chop(1); // 'u' suffix - return FromString<ui64>(str); - } + COLD_BLOCK_BYVALUE + Base::fail("Unexpected ", NCEscape::quote(ch), " in numeric literal"); + COLD_BLOCK_END + } else { + break; + } + check_memory_limit(); + Base::advance(1); + } + + auto str = token_buffer(); + try { + switch (type) { + case number_type::float64: + return FromString<double>(str); + case number_type::int64: + return FromString<i64>(str); + case number_type::uint64: + str.Chop(1); // 'u' suffix + return FromString<ui64>(str); + } Y_UNREACHABLE(); - } catch (const std::exception& err) { - Base::fail(err.what()); - } + } catch (const std::exception& err) { + Base::fail(err.what()); + } } - TStringBuf read_quoted_string() { - auto count_trailing_slashes = [](ui8* begin, ui8* end) { - auto count = size_t{0}; - if (begin < end) { - for (auto p = end - 1; p >= begin && *p == '\\'; --p) { - ++count; - } - } - return count; - }; - - token_buffer_.clear(); - auto& buf = Base::stream().buffer(); - while (true) { + TStringBuf read_quoted_string() { + auto count_trailing_slashes = [](ui8* begin, ui8* end) { + auto count = size_t{0}; + if (begin < end) { + for (auto p = end - 1; p >= begin && *p == '\\'; --p) { + ++count; + } + } + return count; + }; + + token_buffer_.clear(); + auto& buf = Base::stream().buffer(); + while (true) { this->Base::template fill_buffer<false>(); - auto* quote = reinterpret_cast<const ui8*>( - ::memchr(buf.pos(), '"', buf.available())); - if (quote == nullptr) { - token_buffer_.insert( - token_buffer_.end(), - buf.pos(), - buf.end()); - Base::advance(buf.available()); - continue; - } - - token_buffer_.insert( - token_buffer_.end(), - buf.pos(), - quote); - Base::advance(quote - buf.pos() + 1); // +1 for the quote itself - - // We must count the number of '\' at the end of StringValue - // to check if it's not \" - int slash_count = count_trailing_slashes( - token_buffer_.data(), - token_buffer_.data() + token_buffer_.size()); - if (slash_count % 2 == 0) { - break; - } else { - token_buffer_.push_back('"'); - } - check_memory_limit(); + auto* quote = reinterpret_cast<const ui8*>( + ::memchr(buf.pos(), '"', buf.available())); + if (quote == nullptr) { + token_buffer_.insert( + token_buffer_.end(), + buf.pos(), + buf.end()); + Base::advance(buf.available()); + continue; + } + + token_buffer_.insert( + token_buffer_.end(), + buf.pos(), + quote); + Base::advance(quote - buf.pos() + 1); // +1 for the quote itself + + // We must count the number of '\' at the end of StringValue + // to check if it's not \" + int slash_count = count_trailing_slashes( + token_buffer_.data(), + token_buffer_.data() + token_buffer_.size()); + if (slash_count % 2 == 0) { + break; + } else { + token_buffer_.push_back('"'); + } + check_memory_limit(); } - - NCEscape::decode_inplace(token_buffer_); - return token_buffer(); + + NCEscape::decode_inplace(token_buffer_); + return token_buffer(); } - TStringBuf read_unquoted_string() { - token_buffer_.clear(); - while (true) { + TStringBuf read_unquoted_string() { + token_buffer_.clear(); + while (true) { auto ch = this->Base::template get_byte<true>(); - if (isalpha(ch) || isdigit(ch) || - ch == '_' || ch == '-' || ch == '%' || ch == '.') { - token_buffer_.push_back(ch); - } else { - break; - } - check_memory_limit(); - Base::advance(1); - } - return token_buffer(); + if (isalpha(ch) || isdigit(ch) || + ch == '_' || ch == '-' || ch == '%' || ch == '.') { + token_buffer_.push_back(ch); + } else { + break; + } + check_memory_limit(); + Base::advance(1); + } + return token_buffer(); } - ATTRIBUTE(noinline, hot) - TStringBuf read_binary_string() { - auto slength = NVarInt::read<i32>(*this); + ATTRIBUTE(noinline, hot) + TStringBuf read_binary_string() { + auto slength = NVarInt::read<i32>(*this); if (Y_UNLIKELY(slength < 0)) { - COLD_BLOCK_BYVALUE - Base::fail("Negative binary string literal length ", slength); - COLD_BLOCK_END - } - auto length = static_cast<ui32>(slength); + COLD_BLOCK_BYVALUE + Base::fail("Negative binary string literal length ", slength); + COLD_BLOCK_END + } + auto length = static_cast<ui32>(slength); - auto& buf = Base::stream().buffer(); + auto& buf = Base::stream().buffer(); if (Y_LIKELY(buf.available() >= length)) { - auto result = TStringBuf{ - reinterpret_cast<const char*>(buf.pos()), - length}; - Base::advance(length); - return result; - } else { // reading in Buffer - return read_binary_string_fallback(length); - } + auto result = TStringBuf{ + reinterpret_cast<const char*>(buf.pos()), + length}; + Base::advance(length); + return result; + } else { // reading in Buffer + return read_binary_string_fallback(length); + } } - ATTRIBUTE(noinline) - TStringBuf read_binary_string_fallback(size_t length) { - auto& buf = Base::stream().buffer(); - auto needToRead = length; - token_buffer_.clear(); - while (needToRead) { + ATTRIBUTE(noinline) + TStringBuf read_binary_string_fallback(size_t length) { + auto& buf = Base::stream().buffer(); + auto needToRead = length; + token_buffer_.clear(); + while (needToRead) { this->Base::template fill_buffer<false>(); - auto chunk_size = std::min(needToRead, buf.available()); - - token_buffer_.insert( - token_buffer_.end(), - buf.pos(), - buf.pos() + chunk_size); - check_memory_limit(); - needToRead -= chunk_size; - Base::advance(chunk_size); - } - return token_buffer(); + auto chunk_size = std::min(needToRead, buf.available()); + + token_buffer_.insert( + token_buffer_.end(), + buf.pos(), + buf.pos() + chunk_size); + check_memory_limit(); + needToRead -= chunk_size; + Base::advance(chunk_size); + } + return token_buffer(); } percent_scalar read_percent_scalar() { auto throw_incorrect_percent_scalar = [&]() { Base::fail("Incorrect %-literal prefix ", NCEscape::quote(token_buffer())); - }; + }; auto assert_literal = [&](TStringBuf literal) -> void { for (size_t i = 2; i < literal.size(); ++i) { token_buffer_.push_back(this->Base::template get_byte<false>()); - Base::advance(1); + Base::advance(1); if (Y_UNLIKELY(token_buffer_.back() != literal[i])) { throw_incorrect_percent_scalar(); - } - } - }; + } + } + }; - token_buffer_.clear(); + token_buffer_.clear(); token_buffer_.push_back(this->Base::template get_byte<false>()); Base::advance(1); @@ -237,107 +237,107 @@ namespace NYsonPull { default: throw_incorrect_percent_scalar(); } - + Y_UNREACHABLE(); } - i64 read_binary_int64() { - return NVarInt::read<i64>(*this); - } - - ui64 read_binary_uint64() { - return NVarInt::read<ui64>(*this); - } - - double read_binary_double() { - union { - double as_double; - ui8 as_bytes[sizeof(double)]; - } data; - static_assert(sizeof(data) == sizeof(double), "bad union size"); - - auto needToRead = sizeof(double); - - auto& buf = Base::stream().buffer(); - while (needToRead != 0) { - Base::fill_buffer(); - - auto chunk_size = std::min(needToRead, buf.available()); - if (chunk_size == 0) { - Base::fail("Error parsing binary double literal"); - } - std::copy( - buf.pos(), - buf.pos() + chunk_size, - data.as_bytes + (sizeof(double) - needToRead)); - needToRead -= chunk_size; - Base::advance(chunk_size); - } - return data.as_double; - } - - private: - static bool is_space(ui8 ch) { - static const ui8 lookupTable[] = - { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - return lookupTable[ch]; + i64 read_binary_int64() { + return NVarInt::read<i64>(*this); + } + + ui64 read_binary_uint64() { + return NVarInt::read<ui64>(*this); + } + + double read_binary_double() { + union { + double as_double; + ui8 as_bytes[sizeof(double)]; + } data; + static_assert(sizeof(data) == sizeof(double), "bad union size"); + + auto needToRead = sizeof(double); + + auto& buf = Base::stream().buffer(); + while (needToRead != 0) { + Base::fill_buffer(); + + auto chunk_size = std::min(needToRead, buf.available()); + if (chunk_size == 0) { + Base::fail("Error parsing binary double literal"); + } + std::copy( + buf.pos(), + buf.pos() + chunk_size, + data.as_bytes + (sizeof(double) - needToRead)); + needToRead -= chunk_size; + Base::advance(chunk_size); + } + return data.as_double; } - ATTRIBUTE(noinline, cold) - ui8 skip_space_and_get_byte_fallback() { - auto& buf = Base::stream().buffer(); - while (true) { - // FIXME - if (buf.is_empty()) { - if (Base::stream().at_end()) { - return '\0'; - } - Base::fill_buffer(); - } else { - if (!is_space(*buf.pos())) { - break; - } - Base::advance(1); - } + private: + static bool is_space(ui8 ch) { + static const ui8 lookupTable[] = + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + return lookupTable[ch]; + } + + ATTRIBUTE(noinline, cold) + ui8 skip_space_and_get_byte_fallback() { + auto& buf = Base::stream().buffer(); + while (true) { + // FIXME + if (buf.is_empty()) { + if (Base::stream().at_end()) { + return '\0'; + } + Base::fill_buffer(); + } else { + if (!is_space(*buf.pos())) { + break; + } + Base::advance(1); + } } - return Base::get_byte(); - } - - void check_memory_limit() { + return Base::get_byte(); + } + + void check_memory_limit() { if (Y_UNLIKELY(memory_limit_ && token_buffer_.capacity() > *memory_limit_)) { - COLD_BLOCK_BYVALUE - Base::fail( - "Memory limit exceeded while parsing YSON stream: " - "allocated ", - token_buffer_.capacity(), - ", limit ", *memory_limit_); - COLD_BLOCK_END + COLD_BLOCK_BYVALUE + Base::fail( + "Memory limit exceeded while parsing YSON stream: " + "allocated ", + token_buffer_.capacity(), + ", limit ", *memory_limit_); + COLD_BLOCK_END } } - TStringBuf token_buffer() const { - auto* begin = reinterpret_cast<const char*>(token_buffer_.data()); - return {begin, token_buffer_.size()}; - } - }; + TStringBuf token_buffer() const { + auto* begin = reinterpret_cast<const char*>(token_buffer_.data()); + return {begin, token_buffer_.size()}; + } + }; } -} +} diff --git a/library/cpp/yson_pull/detail/macros.h b/library/cpp/yson_pull/detail/macros.h index 7243f9cfe1..bddebe1135 100644 --- a/library/cpp/yson_pull/detail/macros.h +++ b/library/cpp/yson_pull/detail/macros.h @@ -9,16 +9,16 @@ #endif #if defined(__GNUC__) && !defined(__clang__) -#define COLD_BLOCK_BYVALUE [=]() ATTRIBUTE(noinline, cold) { -#define COLD_BLOCK_BYREF [&]() ATTRIBUTE(noinline, cold) { -#define COLD_BLOCK_END \ - } \ - (); +#define COLD_BLOCK_BYVALUE [=]() ATTRIBUTE(noinline, cold) { +#define COLD_BLOCK_BYREF [&]() ATTRIBUTE(noinline, cold) { +#define COLD_BLOCK_END \ + } \ + (); #else // Clang does not support gnu-style attributes on lambda functions yet -#define COLD_BLOCK_BYVALUE [=]() { -#define COLD_BLOCK_BYREF [&]() { -#define COLD_BLOCK_END \ - } \ - (); +#define COLD_BLOCK_BYVALUE [=]() { +#define COLD_BLOCK_BYREF [&]() { +#define COLD_BLOCK_END \ + } \ + (); #endif diff --git a/library/cpp/yson_pull/detail/number.h b/library/cpp/yson_pull/detail/number.h index 5595f55e05..b620082d9d 100644 --- a/library/cpp/yson_pull/detail/number.h +++ b/library/cpp/yson_pull/detail/number.h @@ -2,36 +2,36 @@ #include <util/system/types.h> -namespace NYsonPull { - namespace NDetail { - enum class number_type { - float64, - uint64, - int64 - }; +namespace NYsonPull { + namespace NDetail { + enum class number_type { + float64, + uint64, + int64 + }; - struct number { - number_type type; - union { - double as_float64; - ui64 as_uint64; - i64 as_int64; - } value; + struct number { + number_type type; + union { + double as_float64; + ui64 as_uint64; + i64 as_int64; + } value; - number(double v) { - type = number_type::float64; - value.as_float64 = v; - } + number(double v) { + type = number_type::float64; + value.as_float64 = v; + } - number(i64 v) { - type = number_type::int64; - value.as_int64 = v; - } + number(i64 v) { + type = number_type::int64; + value.as_int64 = v; + } - number(ui64 v) { - type = number_type::uint64; - value.as_uint64 = v; - } - }; + number(ui64 v) { + type = number_type::uint64; + value.as_uint64 = v; + } + }; } -} +} diff --git a/library/cpp/yson_pull/detail/output/buffered.h b/library/cpp/yson_pull/detail/output/buffered.h index 475cf34785..033d8859e0 100644 --- a/library/cpp/yson_pull/detail/output/buffered.h +++ b/library/cpp/yson_pull/detail/output/buffered.h @@ -6,46 +6,46 @@ #include <util/generic/strbuf.h> -namespace NYsonPull { - namespace NDetail { - namespace NOutput { - template <typename T> - class TBuffered: public NYsonPull::NOutput::IStream { - TArrayHolder<ui8> buffer_; - size_t size_; - - public: - TBuffered(size_t buffer_size) - : buffer_{new ui8[buffer_size]} - , size_{buffer_size} { - reset_buffer(); - } - - protected: - void do_flush_buffer(TStringBuf extra) override { - auto& buf = buffer(); - if (!buf.is_empty()) { +namespace NYsonPull { + namespace NDetail { + namespace NOutput { + template <typename T> + class TBuffered: public NYsonPull::NOutput::IStream { + TArrayHolder<ui8> buffer_; + size_t size_; + + public: + TBuffered(size_t buffer_size) + : buffer_{new ui8[buffer_size]} + , size_{buffer_size} { + reset_buffer(); + } + + protected: + void do_flush_buffer(TStringBuf extra) override { + auto& buf = buffer(); + if (!buf.is_empty()) { do_write({reinterpret_cast<const char*>(buf.begin()), buf.used()}); - reset_buffer(); - } - if (extra.size() >= buf.available()) { - do_write(extra); + reset_buffer(); + } + if (extra.size() >= buf.available()) { + do_write(extra); } else if (extra.size() > 0) { - ::memcpy(buf.pos(), extra.data(), extra.size()); - buf.advance(extra.size()); - } - } - - private: - void do_write(TStringBuf data) { - // CRTP dispatch - static_cast<T*>(this)->write(data); - } - - void reset_buffer() { - buffer().reset(buffer_.Get(), buffer_.Get() + size_); - } - }; + ::memcpy(buf.pos(), extra.data(), extra.size()); + buf.advance(extra.size()); + } + } + + private: + void do_write(TStringBuf data) { + // CRTP dispatch + static_cast<T*>(this)->write(data); + } + + void reset_buffer() { + buffer().reset(buffer_.Get(), buffer_.Get() + size_); + } + }; } - } // namespace NDetail -} + } // namespace NDetail +} diff --git a/library/cpp/yson_pull/detail/output/stdio_file.h b/library/cpp/yson_pull/detail/output/stdio_file.h index 03f2b40dc5..0b9cbba894 100644 --- a/library/cpp/yson_pull/detail/output/stdio_file.h +++ b/library/cpp/yson_pull/detail/output/stdio_file.h @@ -8,26 +8,26 @@ #include <cstdio> -namespace NYsonPull { - namespace NDetail { - namespace NOutput { - class TStdioFile: public TBuffered<TStdioFile> { - FILE* file_; +namespace NYsonPull { + namespace NDetail { + namespace NOutput { + class TStdioFile: public TBuffered<TStdioFile> { + FILE* file_; - public: - TStdioFile(FILE* file, size_t buffer_size) - : TBuffered<TStdioFile>(buffer_size) - , file_(file) - { - } + public: + TStdioFile(FILE* file, size_t buffer_size) + : TBuffered<TStdioFile>(buffer_size) + , file_(file) + { + } - void write(TStringBuf data) { - auto nwritten = ::fwrite(data.data(), 1, data.size(), file_); + void write(TStringBuf data) { + auto nwritten = ::fwrite(data.data(), 1, data.size(), file_); if (Y_UNLIKELY(static_cast<size_t>(nwritten) != data.size())) { - throw NException::TSystemError(); - } - } - }; + throw NException::TSystemError(); + } + } + }; } - } // namespace NDetail -} + } // namespace NDetail +} diff --git a/library/cpp/yson_pull/detail/output/stream.h b/library/cpp/yson_pull/detail/output/stream.h index d4810f3353..39f3b33c4c 100644 --- a/library/cpp/yson_pull/detail/output/stream.h +++ b/library/cpp/yson_pull/detail/output/stream.h @@ -27,12 +27,12 @@ namespace NYsonPull { } }; - template <typename TOutput> + template <typename TOutput> class TOwned: public TBuffered<TOwned<TOutput>> { TOutput Output; public: - template <typename... Args> + template <typename... Args> TOwned(size_t buffer_size, Args&&... args) : TBuffered<TOwned>(buffer_size) , Output(std::forward<Args>(args)...) @@ -52,5 +52,5 @@ namespace NYsonPull { } }; } - } // namespace NDetail -} + } // namespace NDetail +} diff --git a/library/cpp/yson_pull/detail/reader.h b/library/cpp/yson_pull/detail/reader.h index 0e02396358..db1dcc2227 100644 --- a/library/cpp/yson_pull/detail/reader.h +++ b/library/cpp/yson_pull/detail/reader.h @@ -8,76 +8,76 @@ #include <util/generic/maybe.h> #include <util/generic/vector.h> -namespace NYsonPull { - namespace NDetail { - /*! \internal */ - //////////////////////////////////////////////////////////////////////////////// - - enum class special_token : ui8 { - // Special values: - // YSON - semicolon = 0, // ; - equals = 1, // = - hash = 2, // # - left_bracket = 3, // [ - right_bracket = 4, // ] - left_brace = 5, // { - right_brace = 6, // } - left_angle = 7, // < - right_angle = 8, // > - }; - - // char_class tree representation: - // Root = xb - // BinaryStringOrOtherSpecialToken = x0b - // BinaryString = 00b - // OtherSpecialToken = 10b - // Other = x1b - // BinaryScalar = xx01b - // BinaryInt64 = 0001b - // BinaryDouble = 0101b - // BinaryFalse = 1001b - // BinaryTrue = 1101b - // Other = xxx11b - // Quote = 00011b - // DigitOrMinus = 00111b - // String = 01011b - // Space = 01111b - // Plus = 10011b - // None = 10111b - // Percent = 11011b - enum class char_class : ui8 { - binary_string = 0, // = 00b - - special_token_mask = 2, // = 10b - semicolon = 2 + (0 << 2), - equals = 2 + (1 << 2), - hash = 2 + (2 << 2), - left_bracket = 2 + (3 << 2), - right_bracket = 2 + (4 << 2), - left_brace = 2 + (5 << 2), - right_brace = 2 + (6 << 2), - left_angle = 2 + (7 << 2), - right_angle = 2 + (8 << 2), - - binary_scalar_mask = 1, - binary_int64 = 1 + (0 << 2), // = 001b - binary_double = 1 + (1 << 2), // = 101b - binary_false = 1 + (2 << 2), // = 1001b - binary_true = 1 + (3 << 2), // = 1101b - binary_uint64 = 1 + (4 << 2), // = 10001b - - other_mask = 3, - quote = 3 + (0 << 2), // = 00011b - number = 3 + (1 << 2), // = 00111b - string = 3 + (2 << 2), // = 01011b - percent = 3 + (6 << 2), // = 11011b - none = 3 + (5 << 2), // = 10111b - }; +namespace NYsonPull { + namespace NDetail { + /*! \internal */ + //////////////////////////////////////////////////////////////////////////////// + + enum class special_token : ui8 { + // Special values: + // YSON + semicolon = 0, // ; + equals = 1, // = + hash = 2, // # + left_bracket = 3, // [ + right_bracket = 4, // ] + left_brace = 5, // { + right_brace = 6, // } + left_angle = 7, // < + right_angle = 8, // > + }; + + // char_class tree representation: + // Root = xb + // BinaryStringOrOtherSpecialToken = x0b + // BinaryString = 00b + // OtherSpecialToken = 10b + // Other = x1b + // BinaryScalar = xx01b + // BinaryInt64 = 0001b + // BinaryDouble = 0101b + // BinaryFalse = 1001b + // BinaryTrue = 1101b + // Other = xxx11b + // Quote = 00011b + // DigitOrMinus = 00111b + // String = 01011b + // Space = 01111b + // Plus = 10011b + // None = 10111b + // Percent = 11011b + enum class char_class : ui8 { + binary_string = 0, // = 00b + + special_token_mask = 2, // = 10b + semicolon = 2 + (0 << 2), + equals = 2 + (1 << 2), + hash = 2 + (2 << 2), + left_bracket = 2 + (3 << 2), + right_bracket = 2 + (4 << 2), + left_brace = 2 + (5 << 2), + right_brace = 2 + (6 << 2), + left_angle = 2 + (7 << 2), + right_angle = 2 + (8 << 2), + + binary_scalar_mask = 1, + binary_int64 = 1 + (0 << 2), // = 001b + binary_double = 1 + (1 << 2), // = 101b + binary_false = 1 + (2 << 2), // = 1001b + binary_true = 1 + (3 << 2), // = 1101b + binary_uint64 = 1 + (4 << 2), // = 10001b + + other_mask = 3, + quote = 3 + (0 << 2), // = 00011b + number = 3 + (1 << 2), // = 00111b + string = 3 + (2 << 2), // = 01011b + percent = 3 + (6 << 2), // = 11011b + none = 3 + (5 << 2), // = 10111b + }; #define CHAR_SUBCLASS(x) (static_cast<ui8>(x) >> 2) - inline char_class get_char_class(ui8 ch) { + inline char_class get_char_class(ui8 ch) { #define NN char_class::none #define BS char_class::binary_string #define BI char_class::binary_int64 @@ -91,70 +91,70 @@ namespace NYsonPull { #define QU char_class::quote #define PC char_class::percent #define TT(name) (static_cast<char_class>( \ - (static_cast<ui8>(special_token::name) << 2) | static_cast<ui8>(char_class::special_token_mask))) - - static constexpr char_class lookup[256] = - { - NN, BS, BI, BD, BF, BT, BU, NN, NN, SP, SP, SP, SP, SP, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - - // 32 - SP, // ' ' - NN, // '!' - QU, // '"' - TT(hash), // '#' - NN, // '$' - PC, // '%' - NN, // '&' - NN, // "'" - NN, // '(' - NN, // ')' - NN, // '*' - NB, // '+' - NN, // ',' - NB, // '-' - NN, // '.' - NN, // '/' - - // 48 - NB, NB, NB, NB, NB, NB, NB, NB, NB, NB, // '0' - '9' - NN, // ':' - TT(semicolon), // ';' - TT(left_angle), // '<' - TT(equals), // '=' - TT(right_angle), // '>' - NN, // '?' - - // 64 - NN, // '@' - ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'A' - 'M' - ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'N' - 'Z' - TT(left_bracket), // '[' - NN, // '\' - TT(right_bracket), // ']' - NN, // '^' - ST, // '_' - - // 96 - NN, // '`' - - ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'a' - 'm' - ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'n' - 'z' - TT(left_brace), // '{' - NN, // '|' - TT(right_brace), // '}' - NN, // '~' - NN, // '^?' non-printable - // 128 - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, - NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN}; + (static_cast<ui8>(special_token::name) << 2) | static_cast<ui8>(char_class::special_token_mask))) + + static constexpr char_class lookup[256] = + { + NN, BS, BI, BD, BF, BT, BU, NN, NN, SP, SP, SP, SP, SP, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + + // 32 + SP, // ' ' + NN, // '!' + QU, // '"' + TT(hash), // '#' + NN, // '$' + PC, // '%' + NN, // '&' + NN, // "'" + NN, // '(' + NN, // ')' + NN, // '*' + NB, // '+' + NN, // ',' + NB, // '-' + NN, // '.' + NN, // '/' + + // 48 + NB, NB, NB, NB, NB, NB, NB, NB, NB, NB, // '0' - '9' + NN, // ':' + TT(semicolon), // ';' + TT(left_angle), // '<' + TT(equals), // '=' + TT(right_angle), // '>' + NN, // '?' + + // 64 + NN, // '@' + ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'A' - 'M' + ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'N' - 'Z' + TT(left_bracket), // '[' + NN, // '\' + TT(right_bracket), // ']' + NN, // '^' + ST, // '_' + + // 96 + NN, // '`' + + ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'a' - 'm' + ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, ST, // 'n' - 'z' + TT(left_brace), // '{' + NN, // '|' + TT(right_brace), // '}' + NN, // '~' + NN, // '^?' non-printable + // 128 + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, + NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN, NN}; #undef NN #undef BS @@ -165,140 +165,140 @@ namespace NYsonPull { #undef ST #undef QU #undef TT - return lookup[ch]; - } - - template <bool EnableLinePositionInfo> - class gen_reader_impl { - enum class state { - delimiter = 0, //! expecting ';' or closing-char ('>', ']', '}') - maybe_value = 1, //! expecting a value or closing-char - maybe_key = 2, //! expecting a key or closing-char - equals = 3, //! expecting '=' (followed by value) - value = 4, //! expecting a value - value_noattr = 5, //! expecting a value w/o attrs (after attrs) - - // by design, rare states have numbers starting from first_rare_state - first_rare_state = 6, - before_begin = first_rare_state, //! before started reading the stream - before_end = first_rare_state + 1, //! Expecting end of stream - after_end = first_rare_state + 2, //! after end of stream - }; - - lexer_base<EnableLinePositionInfo> lexer_; - state state_; - TEvent event_; + return lookup[ch]; + } + + template <bool EnableLinePositionInfo> + class gen_reader_impl { + enum class state { + delimiter = 0, //! expecting ';' or closing-char ('>', ']', '}') + maybe_value = 1, //! expecting a value or closing-char + maybe_key = 2, //! expecting a key or closing-char + equals = 3, //! expecting '=' (followed by value) + value = 4, //! expecting a value + value_noattr = 5, //! expecting a value w/o attrs (after attrs) + + // by design, rare states have numbers starting from first_rare_state + first_rare_state = 6, + before_begin = first_rare_state, //! before started reading the stream + before_end = first_rare_state + 1, //! Expecting end of stream + after_end = first_rare_state + 2, //! after end of stream + }; + + lexer_base<EnableLinePositionInfo> lexer_; + state state_; + TEvent event_; TVector<EEventType> stack_; - EStreamType mode_; - - public: - gen_reader_impl( - NYsonPull::NInput::IStream& buffer, - EStreamType mode, - TMaybe<size_t> memoryLimit = {}) - : lexer_(buffer, memoryLimit) - , state_{state::before_begin} - , mode_{mode} { - } - - const TEvent& last_event() const { - return event_; - } - - ATTRIBUTE(hot) - const TEvent& next_event() { + EStreamType mode_; + + public: + gen_reader_impl( + NYsonPull::NInput::IStream& buffer, + EStreamType mode, + TMaybe<size_t> memoryLimit = {}) + : lexer_(buffer, memoryLimit) + , state_{state::before_begin} + , mode_{mode} { + } + + const TEvent& last_event() const { + return event_; + } + + ATTRIBUTE(hot) + const TEvent& next_event() { if (Y_LIKELY(state_ < state::first_rare_state)) { - // 'hot' handler for in-stream events - next_event_hot(); - } else { - // these events happen no more than once per stream - next_event_cold(); - } - return event_; - } - - private: - ATTRIBUTE(hot) - void next_event_hot() { - auto ch = lexer_.get_byte(); - auto cls = get_char_class(ch); + // 'hot' handler for in-stream events + next_event_hot(); + } else { + // these events happen no more than once per stream + next_event_cold(); + } + return event_; + } + + private: + ATTRIBUTE(hot) + void next_event_hot() { + auto ch = lexer_.get_byte(); + auto cls = get_char_class(ch); if (Y_UNLIKELY(cls == char_class::none)) { - ch = lexer_.skip_space_and_get_byte(); + ch = lexer_.skip_space_and_get_byte(); if (Y_UNLIKELY(ch == NSymbol::eof)) { - handle_eof(); - return; - } - cls = get_char_class(ch); - } - - // states maybe_value/value/value_noattr are distinguished - // later in state_value_special - switch (state_) { - case state::maybe_value: - state_value(ch, cls); - break; - case state::maybe_key: - state_maybe_key(ch, cls); - break; - case state::equals: - state_equals(ch); - break; - case state::value: - state_value(ch, cls); - break; - case state::value_noattr: - state_value(ch, cls); - break; - case state::delimiter: - state_delimiter(ch, cls); - break; - default: + handle_eof(); + return; + } + cls = get_char_class(ch); + } + + // states maybe_value/value/value_noattr are distinguished + // later in state_value_special + switch (state_) { + case state::maybe_value: + state_value(ch, cls); + break; + case state::maybe_key: + state_maybe_key(ch, cls); + break; + case state::equals: + state_equals(ch); + break; + case state::value: + state_value(ch, cls); + break; + case state::value_noattr: + state_value(ch, cls); + break; + case state::delimiter: + state_delimiter(ch, cls); + break; + default: Y_UNREACHABLE(); - } + } } - ATTRIBUTE(noinline, cold) - void next_event_cold() { - switch (state_) { - case state::before_begin: - state_before_begin(); - break; - case state::after_end: - lexer_.fail("Attempted read past stream end"); - case state::before_end: - state_before_end(); - break; - default: + ATTRIBUTE(noinline, cold) + void next_event_cold() { + switch (state_) { + case state::before_begin: + state_before_begin(); + break; + case state::after_end: + lexer_.fail("Attempted read past stream end"); + case state::before_end: + state_before_end(); + break; + default: Y_UNREACHABLE(); - } - } - - //! Present a scalar value for caller - template <typename T> - void yield(T value) { - event_ = TEvent{TScalar{value}}; - } - - //! Present a scalar value with non-scalar tag (i.e. key) - template <typename T> - void yield(EEventType type, T value) { - event_ = TEvent{type, TScalar{value}}; - } - - //! Present a value from number variant - void yield(const number& value) { - switch (value.type) { - case number_type::int64: - yield(value.value.as_int64); - break; - case number_type::uint64: - yield(value.value.as_uint64); - break; - case number_type::float64: - yield(value.value.as_float64); - break; - } - } + } + } + + //! Present a scalar value for caller + template <typename T> + void yield(T value) { + event_ = TEvent{TScalar{value}}; + } + + //! Present a scalar value with non-scalar tag (i.e. key) + template <typename T> + void yield(EEventType type, T value) { + event_ = TEvent{type, TScalar{value}}; + } + + //! Present a value from number variant + void yield(const number& value) { + switch (value.type) { + case number_type::int64: + yield(value.value.as_int64); + break; + case number_type::uint64: + yield(value.value.as_uint64); + break; + case number_type::float64: + yield(value.value.as_float64); + break; + } + } //! Present a value from %-literal variant void yield(const percent_scalar& value) { @@ -312,350 +312,350 @@ namespace NYsonPull { } } - //! Present a value-less event - void yield(EEventType type) { - event_ = TEvent{type}; - } + //! Present a value-less event + void yield(EEventType type) { + event_ = TEvent{type}; + } - //! Push the opening of a paired event - void push(EEventType type) { - stack_.push_back(type); - } + //! Push the opening of a paired event + void push(EEventType type) { + stack_.push_back(type); + } - //! Close the paired_event, verify that delimiters are well-formed - void pop(EEventType first, EEventType last) { + //! Close the paired_event, verify that delimiters are well-formed + void pop(EEventType first, EEventType last) { if (Y_UNLIKELY(stack_.empty() || stack_.back() != first)) { - pop_fail(first, last); - return; - } - stack_.pop_back(); + pop_fail(first, last); + return; + } + stack_.pop_back(); - yield(last); - switch (first) { + yield(last); + switch (first) { case EEventType::BeginList: - next(state::delimiter); - break; + next(state::delimiter); + break; case EEventType::BeginMap: - next(state::delimiter); - break; + next(state::delimiter); + break; case EEventType::BeginAttributes: - next(state::value_noattr); - break; + next(state::value_noattr); + break; case EEventType::BeginStream: - next(state::after_end); - break; + next(state::after_end); + break; - default: + default: Y_UNREACHABLE(); - } + } if (Y_UNLIKELY(mode_ == EStreamType::Node && stack_.size() == 1 && state_ == state::delimiter)) { - next(state::before_end); - } - } - - ATTRIBUTE(noinline, cold) - void pop_fail(EEventType first, EEventType last) { - if (stack_.empty()) { - lexer_.fail("Unpaired events: expected opening '", first, "' for '", last, "', but event stack is empty"); - } else { - lexer_.fail("Unpaired events: expected opening '", first, "' for '", last, "', but '", stack_.back(), "' is found."); - } - } - - //! Transition to new_state - void next(state new_state) { - state_ = new_state; - } - - bool in_map() { + next(state::before_end); + } + } + + ATTRIBUTE(noinline, cold) + void pop_fail(EEventType first, EEventType last) { + if (stack_.empty()) { + lexer_.fail("Unpaired events: expected opening '", first, "' for '", last, "', but event stack is empty"); + } else { + lexer_.fail("Unpaired events: expected opening '", first, "' for '", last, "', but '", stack_.back(), "' is found."); + } + } + + //! Transition to new_state + void next(state new_state) { + state_ = new_state; + } + + bool in_map() { return (stack_.back() == EEventType::BeginMap) || (stack_.back() == EEventType::BeginAttributes) || (stack_.back() == EEventType::BeginStream && mode_ == EStreamType::MapFragment); - } - - ATTRIBUTE(noinline, cold) - void handle_eof() { - switch (state_) { - case state::maybe_value: - case state::maybe_key: - case state::delimiter: - case state::before_end: + } + + ATTRIBUTE(noinline, cold) + void handle_eof() { + switch (state_) { + case state::maybe_value: + case state::maybe_key: + case state::delimiter: + case state::before_end: pop(EEventType::BeginStream, EEventType::EndStream); - return; + return; - default: - lexer_.fail("Unexpected end of stream"); - } - } + default: + lexer_.fail("Unexpected end of stream"); + } + } - ATTRIBUTE(noinline, cold) - void state_before_begin() { + ATTRIBUTE(noinline, cold) + void state_before_begin() { push(EEventType::BeginStream); yield(EEventType::BeginStream); - switch (mode_) { - case EStreamType::Node: - next(state::value); - break; - case EStreamType::ListFragment: - next(state::maybe_value); - break; - case EStreamType::MapFragment: - next(state::maybe_key); - break; - default: + switch (mode_) { + case EStreamType::Node: + next(state::value); + break; + case EStreamType::ListFragment: + next(state::maybe_value); + break; + case EStreamType::MapFragment: + next(state::maybe_key); + break; + default: Y_UNREACHABLE(); - } - } - - ATTRIBUTE(noinline, cold) - void state_before_end() { - auto ch = lexer_.skip_space_and_get_byte(); - if (ch == NSymbol::eof) { - handle_eof(); - } else { - lexer_.fail("Expected stream end, but found ", NCEscape::quote(ch)); - } - } - - ATTRIBUTE(hot) - void state_delimiter(ui8 ch, char_class cls) { + } + } + + ATTRIBUTE(noinline, cold) + void state_before_end() { + auto ch = lexer_.skip_space_and_get_byte(); + if (ch == NSymbol::eof) { + handle_eof(); + } else { + lexer_.fail("Expected stream end, but found ", NCEscape::quote(ch)); + } + } + + ATTRIBUTE(hot) + void state_delimiter(ui8 ch, char_class cls) { if (Y_LIKELY(ch == NSymbol::item_separator)) { - lexer_.advance(1); - next(in_map() ? state::maybe_key : state::maybe_value); - // immediately read next value - next_event_hot(); - return; - } - state_delimiter_fallback(ch, cls); - } - - ATTRIBUTE(noinline, hot) - void state_delimiter_fallback(ui8 ch, char_class cls) { - auto cls_bits = static_cast<ui8>(cls); - if ((cls_bits & 3) == static_cast<ui8>(char_class::special_token_mask)) { - auto token = static_cast<special_token>(cls_bits >> 2); - lexer_.advance(1); - switch (token) { - /* // handled in the fast track + lexer_.advance(1); + next(in_map() ? state::maybe_key : state::maybe_value); + // immediately read next value + next_event_hot(); + return; + } + state_delimiter_fallback(ch, cls); + } + + ATTRIBUTE(noinline, hot) + void state_delimiter_fallback(ui8 ch, char_class cls) { + auto cls_bits = static_cast<ui8>(cls); + if ((cls_bits & 3) == static_cast<ui8>(char_class::special_token_mask)) { + auto token = static_cast<special_token>(cls_bits >> 2); + lexer_.advance(1); + switch (token) { + /* // handled in the fast track case special_token::semicolon: next(in_map()? state::maybe_key : state::maybe_value); // immediately read next value return next_event(); */ - case special_token::right_bracket: + case special_token::right_bracket: pop(EEventType::BeginList, EEventType::EndList); - return; + return; - case special_token::right_brace: + case special_token::right_brace: pop(EEventType::BeginMap, EEventType::EndMap); - return; + return; - case special_token::right_angle: + case special_token::right_angle: pop(EEventType::BeginAttributes, EEventType::EndAttributes); - return; - - default: - break; - } - } - - COLD_BLOCK_BYVALUE - lexer_.fail( - "Unexpected ", NCEscape::quote(ch), ", expected one of ", - NCEscape::quote(NSymbol::item_separator), ", ", - NCEscape::quote(NSymbol::end_list), ", ", - NCEscape::quote(NSymbol::end_map), ", ", - NCEscape::quote(NSymbol::end_attributes)); - COLD_BLOCK_END + return; + + default: + break; + } + } + + COLD_BLOCK_BYVALUE + lexer_.fail( + "Unexpected ", NCEscape::quote(ch), ", expected one of ", + NCEscape::quote(NSymbol::item_separator), ", ", + NCEscape::quote(NSymbol::end_list), ", ", + NCEscape::quote(NSymbol::end_map), ", ", + NCEscape::quote(NSymbol::end_attributes)); + COLD_BLOCK_END } - ATTRIBUTE(noinline, hot) - void state_maybe_key(ui8 ch, char_class cls) { - auto key = TStringBuf{}; - // Keys are always strings, put binary-string key into fast lane + ATTRIBUTE(noinline, hot) + void state_maybe_key(ui8 ch, char_class cls) { + auto key = TStringBuf{}; + // Keys are always strings, put binary-string key into fast lane if (Y_LIKELY(ch == NSymbol::string_marker)) { lexer_.advance(1); - key = lexer_.read_binary_string(); - } else { - switch (cls) { - case char_class::quote: - lexer_.advance(1); - key = lexer_.read_quoted_string(); - break; - - case char_class::string: - key = lexer_.read_unquoted_string(); - break; - - case char_class::right_brace: - lexer_.advance(1); + key = lexer_.read_binary_string(); + } else { + switch (cls) { + case char_class::quote: + lexer_.advance(1); + key = lexer_.read_quoted_string(); + break; + + case char_class::string: + key = lexer_.read_unquoted_string(); + break; + + case char_class::right_brace: + lexer_.advance(1); pop(EEventType::BeginMap, EEventType::EndMap); - return; + return; - case char_class::right_angle: - lexer_.advance(1); + case char_class::right_angle: + lexer_.advance(1); pop(EEventType::BeginAttributes, EEventType::EndAttributes); - return; - - default: - COLD_BLOCK_BYVALUE - lexer_.fail("Unexpected ", NCEscape::quote(ch), ", expected key string"); - COLD_BLOCK_END - } - } - + return; + + default: + COLD_BLOCK_BYVALUE + lexer_.fail("Unexpected ", NCEscape::quote(ch), ", expected key string"); + COLD_BLOCK_END + } + } + yield(EEventType::Key, key); - next(state::equals); - } - - ATTRIBUTE(hot) - void state_equals(ui8 ch) { - // skip '=' + next(state::equals); + } + + ATTRIBUTE(hot) + void state_equals(ui8 ch) { + // skip '=' if (Y_UNLIKELY(ch != NSymbol::key_value_separator)) { COLD_BLOCK_BYVALUE - lexer_.fail("Unexpected ", NCEscape::quote(ch), ", expected ", NCEscape::quote(NSymbol::key_value_separator)); + lexer_.fail("Unexpected ", NCEscape::quote(ch), ", expected ", NCEscape::quote(NSymbol::key_value_separator)); COLD_BLOCK_END - } - lexer_.advance(1); - next(state::value); - // immediately read the following value - // (this symbol yields no result) - next_event_hot(); - } - - ATTRIBUTE(noinline, hot) - void state_value(ui8 ch, char_class cls) { - auto cls_bits = static_cast<ui8>(cls); - if (cls_bits & 1) { // Other = x1b - if (cls_bits & (1 << 1)) { // Other = xxx11b - state_value_text_scalar(cls); - } else { // BinaryScalar = x01b - state_value_binary_scalar(cls); - } - next(state::delimiter); - } else { // BinaryStringOrOtherSpecialToken = x0b - lexer_.advance(1); - if (cls_bits & 1 << 1) { - // special token - auto token = static_cast<special_token>(cls_bits >> 2); - state_value_special(token, ch); - } else { - // binary string - yield(lexer_.read_binary_string()); - next(state::delimiter); - } - } + } + lexer_.advance(1); + next(state::value); + // immediately read the following value + // (this symbol yields no result) + next_event_hot(); } - ATTRIBUTE(noinline) - void state_value_special(special_token token, ui8 ch) { - // Value starters are always accepted values - switch (token) { - case special_token::hash: - yield(TScalar{}); - next(state::delimiter); - return; - - case special_token::left_bracket: + ATTRIBUTE(noinline, hot) + void state_value(ui8 ch, char_class cls) { + auto cls_bits = static_cast<ui8>(cls); + if (cls_bits & 1) { // Other = x1b + if (cls_bits & (1 << 1)) { // Other = xxx11b + state_value_text_scalar(cls); + } else { // BinaryScalar = x01b + state_value_binary_scalar(cls); + } + next(state::delimiter); + } else { // BinaryStringOrOtherSpecialToken = x0b + lexer_.advance(1); + if (cls_bits & 1 << 1) { + // special token + auto token = static_cast<special_token>(cls_bits >> 2); + state_value_special(token, ch); + } else { + // binary string + yield(lexer_.read_binary_string()); + next(state::delimiter); + } + } + } + + ATTRIBUTE(noinline) + void state_value_special(special_token token, ui8 ch) { + // Value starters are always accepted values + switch (token) { + case special_token::hash: + yield(TScalar{}); + next(state::delimiter); + return; + + case special_token::left_bracket: push(EEventType::BeginList); yield(EEventType::BeginList); - next(state::maybe_value); - return; + next(state::maybe_value); + return; - case special_token::left_brace: + case special_token::left_brace: push(EEventType::BeginMap); yield(EEventType::BeginMap); - next(state::maybe_key); - return; + next(state::maybe_key); + return; - default: - break; - } + default: + break; + } - // ...closing-chars are only allowed in maybe_value state - if (state_ == state::maybe_value) { - switch (token) { - case special_token::right_bracket: + // ...closing-chars are only allowed in maybe_value state + if (state_ == state::maybe_value) { + switch (token) { + case special_token::right_bracket: pop(EEventType::BeginList, EEventType::EndList); - return; + return; - case special_token::right_brace: + case special_token::right_brace: pop(EEventType::BeginMap, EEventType::EndMap); - return; - - // right_angle is impossible in maybe_value state - // (only in delimiter, maybe_key) - - default: - break; - } - } - - // attributes are not allowed after attributes (thus, value_noattr state) - if (state_ != state::value_noattr && token == special_token::left_angle) { + return; + + // right_angle is impossible in maybe_value state + // (only in delimiter, maybe_key) + + default: + break; + } + } + + // attributes are not allowed after attributes (thus, value_noattr state) + if (state_ != state::value_noattr && token == special_token::left_angle) { push(EEventType::BeginAttributes); yield(EEventType::BeginAttributes); - next(state::maybe_key); + next(state::maybe_key); return; - } + } - COLD_BLOCK_BYVALUE - lexer_.fail("Unexpected ", NCEscape::quote(ch)); - COLD_BLOCK_END + COLD_BLOCK_BYVALUE + lexer_.fail("Unexpected ", NCEscape::quote(ch)); + COLD_BLOCK_END } - ATTRIBUTE(hot) - void state_value_binary_scalar(char_class cls) { - lexer_.advance(1); - switch (cls) { - case char_class::binary_double: - yield(lexer_.read_binary_double()); - break; + ATTRIBUTE(hot) + void state_value_binary_scalar(char_class cls) { + lexer_.advance(1); + switch (cls) { + case char_class::binary_double: + yield(lexer_.read_binary_double()); + break; - case char_class::binary_int64: - yield(lexer_.read_binary_int64()); - break; + case char_class::binary_int64: + yield(lexer_.read_binary_int64()); + break; - case char_class::binary_uint64: - yield(lexer_.read_binary_uint64()); - break; + case char_class::binary_uint64: + yield(lexer_.read_binary_uint64()); + break; - case char_class::binary_false: - yield(false); - break; + case char_class::binary_false: + yield(false); + break; - case char_class::binary_true: - yield(true); - break; + case char_class::binary_true: + yield(true); + break; - default: + default: Y_UNREACHABLE(); - } - } - - ATTRIBUTE(noinline) - void state_value_text_scalar(char_class cls) { - switch (cls) { - case char_class::quote: - lexer_.advance(1); - yield(lexer_.read_quoted_string()); - break; - - case char_class::number: - yield(lexer_.read_numeric()); - break; - - case char_class::string: - yield(lexer_.read_unquoted_string()); - break; - - case char_class::percent: - lexer_.advance(1); + } + } + + ATTRIBUTE(noinline) + void state_value_text_scalar(char_class cls) { + switch (cls) { + case char_class::quote: + lexer_.advance(1); + yield(lexer_.read_quoted_string()); + break; + + case char_class::number: + yield(lexer_.read_numeric()); + break; + + case char_class::string: + yield(lexer_.read_unquoted_string()); + break; + + case char_class::percent: + lexer_.advance(1); yield(lexer_.read_percent_scalar()); - break; + break; case char_class::none: COLD_BLOCK_BYVALUE @@ -663,15 +663,15 @@ namespace NYsonPull { COLD_BLOCK_END break; - default: + default: Y_UNREACHABLE(); - } - } - }; - - class reader_impl: public gen_reader_impl<false> { - public: - using gen_reader_impl<false>::gen_reader_impl; - }; + } + } + }; + + class reader_impl: public gen_reader_impl<false> { + public: + using gen_reader_impl<false>::gen_reader_impl; + }; } -} +} diff --git a/library/cpp/yson_pull/detail/stream_counter.h b/library/cpp/yson_pull/detail/stream_counter.h index 3b41b27eb6..81146a32f2 100644 --- a/library/cpp/yson_pull/detail/stream_counter.h +++ b/library/cpp/yson_pull/detail/stream_counter.h @@ -4,48 +4,48 @@ #include <cstddef> -namespace NYsonPull { - namespace NDetail { - template <bool EnableLinePositionInfo> - class stream_counter; - - template <> - class stream_counter<true> { - private: - size_t offset_ = 0; - size_t line_ = 1; - size_t column_ = 1; - - public: - TPositionInfo info() const { - return {offset_, line_, column_}; +namespace NYsonPull { + namespace NDetail { + template <bool EnableLinePositionInfo> + class stream_counter; + + template <> + class stream_counter<true> { + private: + size_t offset_ = 0; + size_t line_ = 1; + size_t column_ = 1; + + public: + TPositionInfo info() const { + return {offset_, line_, column_}; + } + + void update(const ui8* begin, const ui8* end) { + offset_ += end - begin; + for (auto current = begin; current != end; ++current) { + ++column_; + if (*current == '\n') { //TODO: memchr + ++line_; + column_ = 1; + } + } } - - void update(const ui8* begin, const ui8* end) { - offset_ += end - begin; - for (auto current = begin; current != end; ++current) { - ++column_; - if (*current == '\n') { //TODO: memchr - ++line_; - column_ = 1; - } - } - } - }; - - template <> - class stream_counter<false> { - private: - size_t offset_ = 0; - - public: - TPositionInfo info() const { - return {offset_, {}, {}}; - } - - void update(const ui8* begin, const ui8* end) { - offset_ += end - begin; - } - }; + }; + + template <> + class stream_counter<false> { + private: + size_t offset_ = 0; + + public: + TPositionInfo info() const { + return {offset_, {}, {}}; + } + + void update(const ui8* begin, const ui8* end) { + offset_ += end - begin; + } + }; } -} +} diff --git a/library/cpp/yson_pull/detail/symbols.h b/library/cpp/yson_pull/detail/symbols.h index fe94bb9c41..dedf437535 100644 --- a/library/cpp/yson_pull/detail/symbols.h +++ b/library/cpp/yson_pull/detail/symbols.h @@ -3,53 +3,53 @@ #include <util/generic/strbuf.h> #include <util/system/types.h> -namespace NYsonPull { - namespace NDetail { - namespace NSymbol { +namespace NYsonPull { + namespace NDetail { + namespace NSymbol { #define SYM(name, value) constexpr ui8 name = value - //! Indicates the beginning of a list. - SYM(begin_list, '['); - //! Indicates the end of a list. - SYM(end_list, ']'); - - //! Indicates the beginning of a map. - SYM(begin_map, '{'); - //! Indicates the end of a map. - SYM(end_map, '}'); - - //! Indicates the beginning of an attribute map. - SYM(begin_attributes, '<'); - //! Indicates the end of an attribute map. - SYM(end_attributes, '>'); - - //! Separates items in lists and pairs in maps or attribute maps. - SYM(item_separator, ';'); - //! Separates keys from values in maps and attribute maps. - SYM(key_value_separator, '='); - - //! Indicates an entity. - SYM(entity, '#'); - //! Indicates end of stream. - SYM(eof, '\0'); - - //! Marks the beginning of a binary string literal. - SYM(string_marker, '\x01'); - //! Marks the beginning of a binary int64 literal. - SYM(int64_marker, '\x02'); - //! Marks the beginning of a binary uint64 literal. - SYM(uint64_marker, '\x06'); - //! Marks the beginning of a binary double literal. - SYM(double_marker, '\x03'); - //! Marks a binary `false' boolean value. - SYM(false_marker, '\x04'); - //! Marks a binary `true' boolean value. - SYM(true_marker, '\x05'); - - //! Text string quote symbol - SYM(quote, '"'); + //! Indicates the beginning of a list. + SYM(begin_list, '['); + //! Indicates the end of a list. + SYM(end_list, ']'); + + //! Indicates the beginning of a map. + SYM(begin_map, '{'); + //! Indicates the end of a map. + SYM(end_map, '}'); + + //! Indicates the beginning of an attribute map. + SYM(begin_attributes, '<'); + //! Indicates the end of an attribute map. + SYM(end_attributes, '>'); + + //! Separates items in lists and pairs in maps or attribute maps. + SYM(item_separator, ';'); + //! Separates keys from values in maps and attribute maps. + SYM(key_value_separator, '='); + + //! Indicates an entity. + SYM(entity, '#'); + //! Indicates end of stream. + SYM(eof, '\0'); + + //! Marks the beginning of a binary string literal. + SYM(string_marker, '\x01'); + //! Marks the beginning of a binary int64 literal. + SYM(int64_marker, '\x02'); + //! Marks the beginning of a binary uint64 literal. + SYM(uint64_marker, '\x06'); + //! Marks the beginning of a binary double literal. + SYM(double_marker, '\x03'); + //! Marks a binary `false' boolean value. + SYM(false_marker, '\x04'); + //! Marks a binary `true' boolean value. + SYM(true_marker, '\x05'); + + //! Text string quote symbol + SYM(quote, '"'); #undef SYM - } - } -} + } + } +} diff --git a/library/cpp/yson_pull/detail/traits.h b/library/cpp/yson_pull/detail/traits.h index 869a3b9c44..d08b261514 100644 --- a/library/cpp/yson_pull/detail/traits.h +++ b/library/cpp/yson_pull/detail/traits.h @@ -2,28 +2,28 @@ #include <type_traits> -namespace NYsonPull { - namespace NDetail { - namespace NTraits { - template <typename T, typename U> - using if_signed = typename std::enable_if< - std::is_signed<T>::value, - U>::type; +namespace NYsonPull { + namespace NDetail { + namespace NTraits { + template <typename T, typename U> + using if_signed = typename std::enable_if< + std::is_signed<T>::value, + U>::type; - template <typename T, typename U> - using if_unsigned = typename std::enable_if< - std::is_unsigned<T>::value, - U>::type; + template <typename T, typename U> + using if_unsigned = typename std::enable_if< + std::is_unsigned<T>::value, + U>::type; - template <typename T> - using to_unsigned = typename std::enable_if< - std::is_signed<T>::value, - typename std::make_unsigned<T>::type>::type; + template <typename T> + using to_unsigned = typename std::enable_if< + std::is_signed<T>::value, + typename std::make_unsigned<T>::type>::type; - template <typename T> - using to_signed = typename std::enable_if< - std::is_unsigned<T>::value, - typename std::make_signed<T>::type>::type; - } - } // namespace NDetail -} + template <typename T> + using to_signed = typename std::enable_if< + std::is_unsigned<T>::value, + typename std::make_signed<T>::type>::type; + } + } // namespace NDetail +} diff --git a/library/cpp/yson_pull/detail/varint.h b/library/cpp/yson_pull/detail/varint.h index 38bf45d925..e75bed9a3e 100644 --- a/library/cpp/yson_pull/detail/varint.h +++ b/library/cpp/yson_pull/detail/varint.h @@ -10,251 +10,251 @@ #include <cstddef> #include <type_traits> -namespace NYsonPull { - namespace NDetail { - namespace NVarInt { - namespace NImpl { - template <typename T> - constexpr inline size_t max_size() { - return (8 * sizeof(T) - 1) / 7 + 1; - } +namespace NYsonPull { + namespace NDetail { + namespace NVarInt { + namespace NImpl { + template <typename T> + constexpr inline size_t max_size() { + return (8 * sizeof(T) - 1) / 7 + 1; + } - template <typename T> - inline size_t write(ui64 value, T&& consume) { - auto stop = false; - auto nwritten = size_t{0}; - while (!stop) { - ++nwritten; - auto byte = static_cast<ui8>(value | 0x80); - value >>= 7; - if (value == 0) { - stop = true; - byte &= 0x7F; - } - consume(byte); - } - return nwritten; - } + template <typename T> + inline size_t write(ui64 value, T&& consume) { + auto stop = false; + auto nwritten = size_t{0}; + while (!stop) { + ++nwritten; + auto byte = static_cast<ui8>(value | 0x80); + value >>= 7; + if (value == 0) { + stop = true; + byte &= 0x7F; + } + consume(byte); + } + return nwritten; + } - template <typename U> - inline bool read_fast(byte_reader<U>& reader, ui64* value) { - auto& buf = reader.stream().buffer(); - auto* ptr = buf.pos(); - ui32 b; + template <typename U> + inline bool read_fast(byte_reader<U>& reader, ui64* value) { + auto& buf = reader.stream().buffer(); + auto* ptr = buf.pos(); + ui32 b; - // Splitting into 32-bit pieces gives better performance on 32-bit - // processors. - ui32 part0 = 0, part1 = 0, part2 = 0; + // Splitting into 32-bit pieces gives better performance on 32-bit + // processors. + ui32 part0 = 0, part1 = 0, part2 = 0; - b = *(ptr++); - part0 = (b & 0x7F); - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part0 |= (b & 0x7F) << 7; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part0 |= (b & 0x7F) << 14; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part0 |= (b & 0x7F) << 21; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part1 = (b & 0x7F); - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part1 |= (b & 0x7F) << 7; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part1 |= (b & 0x7F) << 14; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part1 |= (b & 0x7F) << 21; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part2 = (b & 0x7F); - if (!(b & 0x80)) - goto done; - b = *(ptr++); - part2 |= (b & 0x7F) << 7; - if (!(b & 0x80)) - goto done; + b = *(ptr++); + part0 = (b & 0x7F); + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part0 |= (b & 0x7F) << 7; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part0 |= (b & 0x7F) << 14; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part0 |= (b & 0x7F) << 21; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part1 = (b & 0x7F); + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part1 |= (b & 0x7F) << 7; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part1 |= (b & 0x7F) << 14; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part1 |= (b & 0x7F) << 21; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part2 = (b & 0x7F); + if (!(b & 0x80)) + goto done; + b = *(ptr++); + part2 |= (b & 0x7F) << 7; + if (!(b & 0x80)) + goto done; - // We have overrun the maximum size of a Varint (10 bytes). The data - // must be corrupt. - return false; + // We have overrun the maximum size of a Varint (10 bytes). The data + // must be corrupt. + return false; - done: - reader.advance(ptr - buf.pos()); - *value = (static_cast<ui64>(part0)) | (static_cast<ui64>(part1) << 28) | (static_cast<ui64>(part2) << 56); - return true; - } + done: + reader.advance(ptr - buf.pos()); + *value = (static_cast<ui64>(part0)) | (static_cast<ui64>(part1) << 28) | (static_cast<ui64>(part2) << 56); + return true; + } - template <typename U> - inline bool read_fast(byte_reader<U>& reader, ui32* value) { - // Fast path: We have enough bytes left in the buffer to guarantee that - // this read won't cross the end, so we can skip the checks. - auto& buf = reader.stream().buffer(); - auto* ptr = buf.pos(); - ui32 b; - ui32 result; + template <typename U> + inline bool read_fast(byte_reader<U>& reader, ui32* value) { + // Fast path: We have enough bytes left in the buffer to guarantee that + // this read won't cross the end, so we can skip the checks. + auto& buf = reader.stream().buffer(); + auto* ptr = buf.pos(); + ui32 b; + ui32 result; - b = *(ptr++); - result = (b & 0x7F); - if (!(b & 0x80)) - goto done; - b = *(ptr++); - result |= (b & 0x7F) << 7; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - result |= (b & 0x7F) << 14; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - result |= (b & 0x7F) << 21; - if (!(b & 0x80)) - goto done; - b = *(ptr++); - result |= b << 28; - if (!(b & 0x80)) - goto done; + b = *(ptr++); + result = (b & 0x7F); + if (!(b & 0x80)) + goto done; + b = *(ptr++); + result |= (b & 0x7F) << 7; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + result |= (b & 0x7F) << 14; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + result |= (b & 0x7F) << 21; + if (!(b & 0x80)) + goto done; + b = *(ptr++); + result |= b << 28; + if (!(b & 0x80)) + goto done; - // FIXME - // If the input is larger than 32 bits, we still need to read it all - // and discard the high-order bits. + // FIXME + // If the input is larger than 32 bits, we still need to read it all + // and discard the high-order bits. - for (size_t i = 0; i < max_size<ui64>() - max_size<ui32>(); i++) { - b = *(ptr++); - if (!(b & 0x80)) - goto done; - } + for (size_t i = 0; i < max_size<ui64>() - max_size<ui32>(); i++) { + b = *(ptr++); + if (!(b & 0x80)) + goto done; + } - // We have overrun the maximum size of a Varint (10 bytes). Assume - // the data is corrupt. - return false; + // We have overrun the maximum size of a Varint (10 bytes). Assume + // the data is corrupt. + return false; - done: - reader.advance(ptr - buf.pos()); - *value = result; - return true; - } + done: + reader.advance(ptr - buf.pos()); + *value = result; + return true; + } - template <typename U> - inline bool read_slow(byte_reader<U>& reader, ui64* value) { - // Slow path: This read might cross the end of the buffer, so we - // need to check and refresh the buffer if and when it does. + template <typename U> + inline bool read_slow(byte_reader<U>& reader, ui64* value) { + // Slow path: This read might cross the end of the buffer, so we + // need to check and refresh the buffer if and when it does. - auto& buf = reader.stream().buffer(); - ui64 result = 0; - int count = 0; - ui32 b; + auto& buf = reader.stream().buffer(); + ui64 result = 0; + int count = 0; + ui32 b; - do { - if (count == max_size<ui64>()) { - return false; - } - reader.fill_buffer(); - if (reader.stream().at_end()) { - return false; - } - b = *buf.pos(); - result |= static_cast<ui64>(b & 0x7F) << (7 * count); - reader.advance(1); - ++count; - } while (b & 0x80); + do { + if (count == max_size<ui64>()) { + return false; + } + reader.fill_buffer(); + if (reader.stream().at_end()) { + return false; + } + b = *buf.pos(); + result |= static_cast<ui64>(b & 0x7F) << (7 * count); + reader.advance(1); + ++count; + } while (b & 0x80); - *value = result; - return true; - } + *value = result; + return true; + } - template <typename U> - inline bool read_slow(byte_reader<U>& reader, ui32* value) { - ui64 result; - // fallback to 64-bit reading - if (read_slow(reader, &result) && result <= std::numeric_limits<ui32>::max()) { - *value = static_cast<ui32>(result); - return true; - } + template <typename U> + inline bool read_slow(byte_reader<U>& reader, ui32* value) { + ui64 result; + // fallback to 64-bit reading + if (read_slow(reader, &result) && result <= std::numeric_limits<ui32>::max()) { + *value = static_cast<ui32>(result); + return true; + } - return false; - } + return false; + } - // Following functions is an adaptation - // of Protobuf code from coded_stream.cc - template <typename T, typename U> - inline bool read_dispatch(byte_reader<U>& reader, T* value) { - auto& buf = reader.stream().buffer(); + // Following functions is an adaptation + // of Protobuf code from coded_stream.cc + template <typename T, typename U> + inline bool read_dispatch(byte_reader<U>& reader, T* value) { + auto& buf = reader.stream().buffer(); // NOTE: checking for 64-bit max_size(), since 32-bit // read_fast() might fallback to 64-bit reading if (buf.available() >= max_size<ui64>() || - // Optimization: If the Varint ends at exactly the end of the buffer, - // we can detect that and still use the fast path. - (!buf.is_empty() && !(buf.end()[-1] & 0x80))) - { - return read_fast(reader, value); - } else { - // Really slow case: we will incur the cost of an extra function call here, - // but moving this out of line reduces the size of this function, which - // improves the common case. In micro benchmarks, this is worth about 10-15% - return read_slow(reader, value); - } - } + // Optimization: If the Varint ends at exactly the end of the buffer, + // we can detect that and still use the fast path. + (!buf.is_empty() && !(buf.end()[-1] & 0x80))) + { + return read_fast(reader, value); + } else { + // Really slow case: we will incur the cost of an extra function call here, + // but moving this out of line reduces the size of this function, which + // improves the common case. In micro benchmarks, this is worth about 10-15% + return read_slow(reader, value); + } + } - } + } - // Various functions to read/write varints. + // Various functions to read/write varints. - // Returns the number of bytes written. - template <typename T> - inline NTraits::if_unsigned<T, size_t> write(ui8* data, T value) { - return NImpl::write( - static_cast<ui64>(value), - [&](ui8 byte) { *data++ = byte; }); - } + // Returns the number of bytes written. + template <typename T> + inline NTraits::if_unsigned<T, size_t> write(ui8* data, T value) { + return NImpl::write( + static_cast<ui64>(value), + [&](ui8 byte) { *data++ = byte; }); + } - template <typename T> - inline NTraits::if_signed<T, size_t> write(ui8* data, T value) { - return NImpl::write( - static_cast<ui64>(NZigZag::encode(value)), - [&](ui8 byte) { *data++ = byte; }); - } + template <typename T> + inline NTraits::if_signed<T, size_t> write(ui8* data, T value) { + return NImpl::write( + static_cast<ui64>(NZigZag::encode(value)), + [&](ui8 byte) { *data++ = byte; }); + } - template <typename T, typename U> - inline void write(byte_writer<U>& stream, T value) { - ui8 data[NImpl::max_size<T>()]; - auto size = write(data, value); - stream.write(data, size); - } + template <typename T, typename U> + inline void write(byte_writer<U>& stream, T value) { + ui8 data[NImpl::max_size<T>()]; + auto size = write(data, value); + stream.write(data, size); + } - template <typename T, typename U> - inline NTraits::if_unsigned<T, T> read(byte_reader<U>& reader) { - auto value = T{}; - auto& buf = reader.stream().buffer(); - if (!buf.is_empty() && *buf.pos() < 0x80) { - value = *buf.pos(); - reader.advance(1); - return value; - } + template <typename T, typename U> + inline NTraits::if_unsigned<T, T> read(byte_reader<U>& reader) { + auto value = T{}; + auto& buf = reader.stream().buffer(); + if (!buf.is_empty() && *buf.pos() < 0x80) { + value = *buf.pos(); + reader.advance(1); + return value; + } if (Y_UNLIKELY(!NImpl::read_dispatch(reader, &value))) { - reader.fail("Error parsing varint value"); - } - return value; - } + reader.fail("Error parsing varint value"); + } + return value; + } - template <typename T, typename U> - inline NTraits::if_signed<T, T> read(byte_reader<U>& reader) { - return NZigZag::decode( - read<NTraits::to_unsigned<T>>(reader)); - } - } - } // namespace NDetail -} + template <typename T, typename U> + inline NTraits::if_signed<T, T> read(byte_reader<U>& reader) { + return NZigZag::decode( + read<NTraits::to_unsigned<T>>(reader)); + } + } + } // namespace NDetail +} diff --git a/library/cpp/yson_pull/detail/writer.h b/library/cpp/yson_pull/detail/writer.h index b24b994292..2c9ab68199 100644 --- a/library/cpp/yson_pull/detail/writer.h +++ b/library/cpp/yson_pull/detail/writer.h @@ -18,439 +18,439 @@ #include <cmath> -namespace NYsonPull { - namespace NDetail { - class writer: public IConsumer { - enum class state { - maybe_key, - maybe_value, - value, - value_noattr, - before_begin, - before_end, - after_end, - }; - - byte_writer<stream_counter<false>> stream_; +namespace NYsonPull { + namespace NDetail { + class writer: public IConsumer { + enum class state { + maybe_key, + maybe_value, + value, + value_noattr, + before_begin, + before_end, + after_end, + }; + + byte_writer<stream_counter<false>> stream_; TVector<EEventType> stack_; - bool need_item_separator_ = false; - EStreamType mode_ = EStreamType::ListFragment; - state state_ = state::before_begin; + bool need_item_separator_ = false; + EStreamType mode_ = EStreamType::ListFragment; + state state_ = state::before_begin; - public: + public: void OnBeginStream() override { update_state(EEventType::BeginStream); - } + } void OnEndStream() override { update_state(EEventType::EndStream); - stream_.flush_buffer(); - } + stream_.flush_buffer(); + } void OnBeginList() override { - begin_node(); - write(NSymbol::begin_list); + begin_node(); + write(NSymbol::begin_list); update_state(EEventType::BeginList); - begin_collection(collection_type::list); - } + begin_collection(collection_type::list); + } void OnEndList() override { update_state(EEventType::EndList); - end_collection(collection_type::list); - write(NSymbol::end_list); - end_node(); - } + end_collection(collection_type::list); + write(NSymbol::end_list); + end_node(); + } void OnBeginMap() override { - begin_node(); - write(NSymbol::begin_map); + begin_node(); + write(NSymbol::begin_map); update_state(EEventType::BeginMap); - begin_collection(collection_type::map); - } + begin_collection(collection_type::map); + } void OnEndMap() override { update_state(EEventType::EndMap); - end_collection(collection_type::map); - write(NSymbol::end_map); - end_node(); - } + end_collection(collection_type::map); + write(NSymbol::end_map); + end_node(); + } void OnBeginAttributes() override { - begin_node(); - write(NSymbol::begin_attributes); + begin_node(); + write(NSymbol::begin_attributes); update_state(EEventType::BeginAttributes); - begin_collection(collection_type::attributes); - } + begin_collection(collection_type::attributes); + } void OnEndAttributes() override { update_state(EEventType::EndAttributes); - end_collection(collection_type::attributes); - write(NSymbol::end_attributes); - // no end_node - } + end_collection(collection_type::attributes); + write(NSymbol::end_attributes); + // no end_node + } void OnEntity() override { begin_node(); update_state(EEventType::Scalar); - write(NSymbol::entity); - end_node(); - } - - protected: - enum class collection_type { - list, - map, - attributes, - }; - - writer(NYsonPull::NOutput::IStream& stream, EStreamType mode) - : stream_(stream) - , mode_{mode} { - } - - bool need_item_separator() const { - return need_item_separator_; - } - void need_item_separator(bool value) { - need_item_separator_ = value; - } - - size_t depth() const { - Y_ASSERT(!stack_.empty()); - if (mode_ == EStreamType::Node) { - return stack_.size() - 1; - } else { - return stack_.size() - 2; - } - } - EStreamType mode() const { - return mode_; - } - - void write(ui8 c) { - stream_.write(c); - } - - void write(TStringBuf value) { - write_raw(value.data(), value.size()); - } - - void write_raw(const void* ptr, size_t len) { - stream_.write(static_cast<const ui8*>(ptr), len); - } - - template <typename T> - void write_varint(T value) { - NVarInt::write(stream_, value); - } - - void write_escaped_string(TStringBuf value) { - write(NSymbol::quote); - NCEscape::encode(stream_, value); - write(NSymbol::quote); - } - - void push(EEventType type) { - stack_.push_back(type); - } - - void pop(EEventType type) { - if (stack_.empty()) { - fail("Unpaired events: empty event stack"); + write(NSymbol::entity); + end_node(); + } + + protected: + enum class collection_type { + list, + map, + attributes, + }; + + writer(NYsonPull::NOutput::IStream& stream, EStreamType mode) + : stream_(stream) + , mode_{mode} { + } + + bool need_item_separator() const { + return need_item_separator_; + } + void need_item_separator(bool value) { + need_item_separator_ = value; + } + + size_t depth() const { + Y_ASSERT(!stack_.empty()); + if (mode_ == EStreamType::Node) { + return stack_.size() - 1; + } else { + return stack_.size() - 2; + } + } + EStreamType mode() const { + return mode_; + } + + void write(ui8 c) { + stream_.write(c); + } + + void write(TStringBuf value) { + write_raw(value.data(), value.size()); + } + + void write_raw(const void* ptr, size_t len) { + stream_.write(static_cast<const ui8*>(ptr), len); + } + + template <typename T> + void write_varint(T value) { + NVarInt::write(stream_, value); + } + + void write_escaped_string(TStringBuf value) { + write(NSymbol::quote); + NCEscape::encode(stream_, value); + write(NSymbol::quote); + } + + void push(EEventType type) { + stack_.push_back(type); + } + + void pop(EEventType type) { + if (stack_.empty()) { + fail("Unpaired events: empty event stack"); } - if (stack_.back() != type) { - fail("Unpaired events: expected ", type, ", got ", stack_.back()); + if (stack_.back() != type) { + fail("Unpaired events: expected ", type, ", got ", stack_.back()); } - stack_.pop_back(); - } + stack_.pop_back(); + } - void update_state(EEventType event) { - switch (state_) { - case state::before_begin: + void update_state(EEventType event) { + switch (state_) { + case state::before_begin: if (event != EEventType::BeginStream) { - fail("Expected begin_stream, got ", event); - } - begin_stream(); - return; + fail("Expected begin_stream, got ", event); + } + begin_stream(); + return; - case state::before_end: + case state::before_end: if (event != EEventType::EndStream) { - fail("Expected end_stream, got ", event); - } + fail("Expected end_stream, got ", event); + } end_stream(); return; - case state::after_end: - fail("Attempted write past stream end"); + case state::after_end: + fail("Attempted write past stream end"); - case state::maybe_key: + case state::maybe_key: if (event == EEventType::Key) { - state_ = state::value; - return; - } + state_ = state::value; + return; + } - switch (event) { + switch (event) { case EEventType::EndStream: - end_stream(); - return; + end_stream(); + return; case EEventType::EndMap: pop(EEventType::BeginMap); - next_state(); - return; + next_state(); + return; case EEventType::EndAttributes: pop(EEventType::BeginAttributes); - state_ = state::value_noattr; - return; + state_ = state::value_noattr; + return; - default: - fail("Unexpected event ", event, " in maybe_key"); - } - break; + default: + fail("Unexpected event ", event, " in maybe_key"); + } + break; - case state::maybe_value: - switch (event) { + case state::maybe_value: + switch (event) { case EEventType::EndList: pop(EEventType::BeginList); - next_state(); - return; + next_state(); + return; case EEventType::EndStream: - end_stream(); - return; + end_stream(); + return; - default: - break; - } + default: + break; + } [[fallthrough]]; - case state::value: + case state::value: if (event == EEventType::BeginAttributes) { push(EEventType::BeginAttributes); - next_state(); - return; - } + next_state(); + return; + } [[fallthrough]]; - case state::value_noattr: - switch (event) { + case state::value_noattr: + switch (event) { case EEventType::Scalar: - next_state(); - return; - + next_state(); + return; + case EEventType::BeginList: push(EEventType::BeginList); - next_state(); - return; - + next_state(); + return; + case EEventType::BeginMap: push(EEventType::BeginMap); - next_state(); - return; - - default: - fail("Unexpected event ", event, " (in value_*)"); - } - break; + next_state(); + return; + + default: + fail("Unexpected event ", event, " (in value_*)"); + } + break; } - } + } - void next_state() { - Y_ASSERT(!stack_.empty()); - switch (stack_.back()) { + void next_state() { + Y_ASSERT(!stack_.empty()); + switch (stack_.back()) { case EEventType::BeginMap: case EEventType::BeginAttributes: - state_ = state::maybe_key; - break; + state_ = state::maybe_key; + break; case EEventType::BeginList: - state_ = state::maybe_value; - break; + state_ = state::maybe_value; + break; case EEventType::BeginStream: - state_ = state::before_end; - break; + state_ = state::before_end; + break; - default: + default: Y_UNREACHABLE(); - } - } + } + } - void begin_stream() { + void begin_stream() { push(EEventType::BeginStream); - switch (mode_) { - case EStreamType::ListFragment: + switch (mode_) { + case EStreamType::ListFragment: push(EEventType::BeginList); - state_ = state::maybe_value; - break; + state_ = state::maybe_value; + break; - case EStreamType::MapFragment: + case EStreamType::MapFragment: push(EEventType::BeginMap); - state_ = state::maybe_key; - break; - - case EStreamType::Node: - state_ = state::value; - break; - } - } - - void end_stream() { - switch (mode_) { - case EStreamType::ListFragment: + state_ = state::maybe_key; + break; + + case EStreamType::Node: + state_ = state::value; + break; + } + } + + void end_stream() { + switch (mode_) { + case EStreamType::ListFragment: pop(EEventType::BeginList); - break; + break; - case EStreamType::MapFragment: + case EStreamType::MapFragment: pop(EEventType::BeginMap); - break; + break; - case EStreamType::Node: - break; - } + case EStreamType::Node: + break; + } pop(EEventType::BeginStream); - state_ = state::after_end; - } - - virtual void begin_node() { - if (need_item_separator_) { - write(NSymbol::item_separator); - } - } - - virtual void end_node() { - need_item_separator_ = true; - } - - virtual void begin_key() { - begin_node(); - } - - virtual void end_key() { - need_item_separator_ = false; - write(NSymbol::key_value_separator); - } - - virtual void begin_collection(collection_type type) { - Y_UNUSED(type); - need_item_separator_ = false; - } - - virtual void end_collection(collection_type type) { - need_item_separator_ = (type != collection_type::attributes); - } - - template <typename... Args> - ATTRIBUTE(noinline, cold) - void fail[[noreturn]](const char* msg, Args&&... args) { - auto formatted_message = format_string( - msg, - std::forward<Args>(args)...); - throw NException::TBadOutput( - formatted_message, - stream_.counter().info()); - } - }; + state_ = state::after_end; + } + + virtual void begin_node() { + if (need_item_separator_) { + write(NSymbol::item_separator); + } + } + + virtual void end_node() { + need_item_separator_ = true; + } + + virtual void begin_key() { + begin_node(); + } + + virtual void end_key() { + need_item_separator_ = false; + write(NSymbol::key_value_separator); + } + + virtual void begin_collection(collection_type type) { + Y_UNUSED(type); + need_item_separator_ = false; + } + + virtual void end_collection(collection_type type) { + need_item_separator_ = (type != collection_type::attributes); + } + + template <typename... Args> + ATTRIBUTE(noinline, cold) + void fail[[noreturn]](const char* msg, Args&&... args) { + auto formatted_message = format_string( + msg, + std::forward<Args>(args)...); + throw NException::TBadOutput( + formatted_message, + stream_.counter().info()); + } + }; class TBinaryWriterImpl final: public writer { - public: + public: TBinaryWriterImpl(NYsonPull::NOutput::IStream& stream, EStreamType mode) - : writer(stream, mode) - { - } + : writer(stream, mode) + { + } void OnScalarBoolean(bool value) override { update_state(EEventType::Scalar); - begin_node(); - write(value ? NSymbol::true_marker : NSymbol::false_marker); - end_node(); - } + begin_node(); + write(value ? NSymbol::true_marker : NSymbol::false_marker); + end_node(); + } void OnScalarInt64(i64 value) override { update_state(EEventType::Scalar); - begin_node(); - write(NSymbol::int64_marker); - write_varint(value); - end_node(); - } + begin_node(); + write(NSymbol::int64_marker); + write_varint(value); + end_node(); + } void OnScalarUInt64(ui64 value) override { update_state(EEventType::Scalar); - begin_node(); - write(NSymbol::uint64_marker); - write_varint(value); - end_node(); - } + begin_node(); + write(NSymbol::uint64_marker); + write_varint(value); + end_node(); + } void OnScalarFloat64(double value) override { update_state(EEventType::Scalar); - begin_node(); - write(NSymbol::double_marker); - write_raw(&value, sizeof value); - end_node(); - } + begin_node(); + write(NSymbol::double_marker); + write_raw(&value, sizeof value); + end_node(); + } void OnScalarString(TStringBuf value) override { update_state(EEventType::Scalar); - begin_node(); - write(NSymbol::string_marker); - write_varint(static_cast<i32>(value.size())); - write_raw(value.data(), value.size()); - end_node(); - } + begin_node(); + write(NSymbol::string_marker); + write_varint(static_cast<i32>(value.size())); + write_raw(value.data(), value.size()); + end_node(); + } void OnKey(TStringBuf name) override { update_state(EEventType::Key); - begin_key(); - write(NSymbol::string_marker); - write_varint(static_cast<i32>(name.size())); - write_raw(name.data(), name.size()); - end_key(); - } - }; + begin_key(); + write(NSymbol::string_marker); + write_varint(static_cast<i32>(name.size())); + write_raw(name.data(), name.size()); + end_key(); + } + }; class TTextWriterImpl: public writer { - public: + public: TTextWriterImpl(NYsonPull::NOutput::IStream& stream, EStreamType mode) - : writer(stream, mode) - { - } + : writer(stream, mode) + { + } void OnScalarBoolean(bool value) override { update_state(EEventType::Scalar); - begin_node(); + begin_node(); write(value ? percent_scalar::true_literal : percent_scalar::false_literal); - end_node(); - } + end_node(); + } void OnScalarInt64(i64 value) override { update_state(EEventType::Scalar); - char buf[32]; + char buf[32]; auto len = ::snprintf(buf, sizeof(buf), "%" PRIi64, value); - begin_node(); - write_raw(buf, len); - end_node(); - } + begin_node(); + write_raw(buf, len); + end_node(); + } void OnScalarUInt64(ui64 value) override { update_state(EEventType::Scalar); - char buf[32]; + char buf[32]; auto len = ::snprintf(buf, sizeof(buf), "%" PRIu64, value); - begin_node(); - write_raw(buf, len); - write('u'); - end_node(); - } + begin_node(); + write_raw(buf, len); + write('u'); + end_node(); + } void OnScalarFloat64(double value) override { update_state(EEventType::Scalar); @@ -469,98 +469,98 @@ namespace NYsonPull { write(percent_scalar::negative_inf_literal); } - end_node(); - } + end_node(); + } void OnScalarString(TStringBuf value) override { update_state(EEventType::Scalar); - begin_node(); - write_escaped_string(value); - end_node(); - } + begin_node(); + write_escaped_string(value); + end_node(); + } void OnKey(TStringBuf name) override { update_state(EEventType::Key); - begin_key(); - write_escaped_string(name); - end_key(); - } - - protected: - void begin_node() override { - if (need_item_separator()) { - write(NSymbol::item_separator); - write(' '); - } - } - - void end_node() override { - if (mode() != EStreamType::Node && depth() == 0) { - write(NSymbol::item_separator); - write('\n'); - need_item_separator(false); - } else { - writer::end_node(); - } - } - - void end_key() override { - write(' '); - writer::end_key(); - write(' '); - } - }; + begin_key(); + write_escaped_string(name); + end_key(); + } + + protected: + void begin_node() override { + if (need_item_separator()) { + write(NSymbol::item_separator); + write(' '); + } + } + + void end_node() override { + if (mode() != EStreamType::Node && depth() == 0) { + write(NSymbol::item_separator); + write('\n'); + need_item_separator(false); + } else { + writer::end_node(); + } + } + + void end_key() override { + write(' '); + writer::end_key(); + write(' '); + } + }; class TPrettyWriterImpl final: public TTextWriterImpl { - size_t indent_size_; + size_t indent_size_; - public: + public: TPrettyWriterImpl( - NYsonPull::NOutput::IStream& stream, - EStreamType mode, - size_t indent_size) + NYsonPull::NOutput::IStream& stream, + EStreamType mode, + size_t indent_size) : TTextWriterImpl(stream, mode) - , indent_size_{indent_size} { - } - - protected: - void begin_node() override { - if (need_item_separator()) { - write(NSymbol::item_separator); - newline(); - } - } - - void begin_collection(collection_type type) override { + , indent_size_{indent_size} { + } + + protected: + void begin_node() override { + if (need_item_separator()) { + write(NSymbol::item_separator); + newline(); + } + } + + void begin_collection(collection_type type) override { TTextWriterImpl::begin_collection(type); - newline(); - } + newline(); + } - void end_collection(collection_type type) override { + void end_collection(collection_type type) override { TTextWriterImpl::end_collection(type); - newline(); - } - - void newline() { - write('\n'); - indent(depth()); - } - - void indent(size_t count) { - for (size_t i = 0; i < count * indent_size_; ++i) { - write(' '); - } - } - }; - - template <typename T, typename... Args> - NYsonPull::TWriter make_writer( - THolder<NYsonPull::NOutput::IStream> stream, - Args&&... args) { - auto impl = MakeHolder<T>(*stream, std::forward<Args>(args)...); - return NYsonPull::TWriter(std::move(stream), std::move(impl)); + newline(); + } + + void newline() { + write('\n'); + indent(depth()); + } + + void indent(size_t count) { + for (size_t i = 0; i < count * indent_size_; ++i) { + write(' '); + } + } + }; + + template <typename T, typename... Args> + NYsonPull::TWriter make_writer( + THolder<NYsonPull::NOutput::IStream> stream, + Args&&... args) { + auto impl = MakeHolder<T>(*stream, std::forward<Args>(args)...); + return NYsonPull::TWriter(std::move(stream), std::move(impl)); } } -} +} diff --git a/library/cpp/yson_pull/detail/zigzag.h b/library/cpp/yson_pull/detail/zigzag.h index 98fcac0e9f..4fca549f49 100644 --- a/library/cpp/yson_pull/detail/zigzag.h +++ b/library/cpp/yson_pull/detail/zigzag.h @@ -2,23 +2,23 @@ #include "traits.h" -namespace NYsonPull { - namespace NDetail { - namespace NZigZag { - //! Functions that provide coding of integers with property: 0 <= f(x) <= 2 * |x| +namespace NYsonPull { + namespace NDetail { + namespace NZigZag { + //! Functions that provide coding of integers with property: 0 <= f(x) <= 2 * |x| template <typename TSigned> inline NTraits::to_unsigned<TSigned> encode(TSigned x) { using TUnsigned = NTraits::to_unsigned<TSigned>; constexpr auto rshift = sizeof(TSigned) * 8 - 1; return (static_cast<TUnsigned>(x) << 1) ^ static_cast<TUnsigned>(x >> rshift); - } + } template <typename TUnsigned> inline NTraits::to_signed<TUnsigned> decode(TUnsigned x) { using TSigned = NTraits::to_signed<TUnsigned>; return static_cast<TSigned>(x >> 1) ^ -static_cast<TSigned>(x & 1); - } - } - } // namespace NDetail -} + } + } + } // namespace NDetail +} diff --git a/library/cpp/yson_pull/event.cpp b/library/cpp/yson_pull/event.cpp index b7ede494b6..efe88a4e42 100644 --- a/library/cpp/yson_pull/event.cpp +++ b/library/cpp/yson_pull/event.cpp @@ -6,7 +6,7 @@ using namespace NYsonPull; -template <> +template <> void Out<TEvent>(IOutputStream& out, const TEvent& value) { out << '(' << value.Type(); if (value.Type() == EEventType::Scalar) { diff --git a/library/cpp/yson_pull/event.h b/library/cpp/yson_pull/event.h index b41d5ea3b5..c141b06889 100644 --- a/library/cpp/yson_pull/event.h +++ b/library/cpp/yson_pull/event.h @@ -8,22 +8,22 @@ #include <util/system/yassert.h> namespace NYsonPull { - //! A well-formed decoded YSON stream can be described by the following grammar: - //! - //! STREAM[node] ::= begin_stream VALUE end_stream - //! STREAM[list_fragment] ::= begin_stream LIST_FRAGMENT end_stream - //! STREAM[map_fragment] ::= begin_stream MAP_FRAGMENT end_stream - //! LIST_FRAGMENT ::= { VALUE; } - //! MAP_FRAGMENT ::= { KEY VALUE; } - //! KEY ::= key(String) - //! VALUE ::= VALUE_NOATTR | ATTRIBUTES VALUE_NOATTR - //! ATTRIBUTES ::= begin_attributes MAP_FRAGMENT end_attributes - //! VALUE_NOATTR ::= scalar(Scalar) | LIST | MAP - //! LIST ::= begin_list LIST_FRAGMENT end_list - //! MAP ::= begin_map MAP_FRAGMENT end_map + //! A well-formed decoded YSON stream can be described by the following grammar: + //! + //! STREAM[node] ::= begin_stream VALUE end_stream + //! STREAM[list_fragment] ::= begin_stream LIST_FRAGMENT end_stream + //! STREAM[map_fragment] ::= begin_stream MAP_FRAGMENT end_stream + //! LIST_FRAGMENT ::= { VALUE; } + //! MAP_FRAGMENT ::= { KEY VALUE; } + //! KEY ::= key(String) + //! VALUE ::= VALUE_NOATTR | ATTRIBUTES VALUE_NOATTR + //! ATTRIBUTES ::= begin_attributes MAP_FRAGMENT end_attributes + //! VALUE_NOATTR ::= scalar(Scalar) | LIST | MAP + //! LIST ::= begin_list LIST_FRAGMENT end_list + //! MAP ::= begin_map MAP_FRAGMENT end_map - //! \brief YSON event type tag. Corresponds to YSON grammar. - enum class EEventType { + //! \brief YSON event type tag. Corresponds to YSON grammar. + enum class EEventType { BeginStream = YSON_EVENT_BEGIN_STREAM, EndStream = YSON_EVENT_END_STREAM, BeginList = YSON_EVENT_BEGIN_LIST, @@ -34,52 +34,52 @@ namespace NYsonPull { EndAttributes = YSON_EVENT_END_ATTRIBUTES, Key = YSON_EVENT_KEY, Scalar = YSON_EVENT_SCALAR, - }; + }; - //! \brief YSON event variant type. - class TEvent { + //! \brief YSON event variant type. + class TEvent { EEventType Type_; TScalar Value_; - public: - //! \brief Construct a tag-only event. + public: + //! \brief Construct a tag-only event. explicit constexpr TEvent(EEventType type = EEventType::BeginStream) : Type_{type} { - } + } - //! \brief Construct a tag+value event. - //! - //! Only \p EEventType::key is meaningful. - constexpr TEvent(EEventType type, const TScalar& value) + //! \brief Construct a tag+value event. + //! + //! Only \p EEventType::key is meaningful. + constexpr TEvent(EEventType type, const TScalar& value) : Type_{type} , Value_{value} { - } + } - //! \brief Construct a \p EEventType::scalar event. - explicit constexpr TEvent(const TScalar& value) + //! \brief Construct a \p EEventType::scalar event. + explicit constexpr TEvent(const TScalar& value) : Type_{EEventType::Scalar} , Value_{value} { - } + } EEventType Type() const { return Type_; - } + } - //! \brief Get TScalar value. - //! - //! Undefined behaviour when event type is not \p EEventType::scalar. + //! \brief Get TScalar value. + //! + //! Undefined behaviour when event type is not \p EEventType::scalar. const TScalar& AsScalar() const { Y_ASSERT(Type_ == EEventType::Scalar || Type_ == EEventType::Key); return Value_; - } + } - //! \brief Get string value. - //! - //! Undefined behaviour when event type is not \p EEventType::key. + //! \brief Get string value. + //! + //! Undefined behaviour when event type is not \p EEventType::key. TStringBuf AsString() const { Y_ASSERT(Type_ == EEventType::Key || (Type_ == EEventType::Scalar && Value_.Type() == EScalarType::String)); return Value_.AsString(); - } - }; + } + }; -} +} diff --git a/library/cpp/yson_pull/exceptions.cpp b/library/cpp/yson_pull/exceptions.cpp index e1d68493e7..ac482b567c 100644 --- a/library/cpp/yson_pull/exceptions.cpp +++ b/library/cpp/yson_pull/exceptions.cpp @@ -36,9 +36,9 @@ const char* TBadStream::what() const noexcept { return FormattedMessage_.c_str(); } -NYsonPull::NException::TSystemError::TSystemError() +NYsonPull::NException::TSystemError::TSystemError() : SavedErrno_{errno} { -} +} const char* NYsonPull::NException::TSystemError::what() const noexcept { return ::strerror(SavedErrno_); diff --git a/library/cpp/yson_pull/exceptions.h b/library/cpp/yson_pull/exceptions.h index ebfed950a5..f71b711922 100644 --- a/library/cpp/yson_pull/exceptions.h +++ b/library/cpp/yson_pull/exceptions.h @@ -7,53 +7,53 @@ #include <stdexcept> #include <string> -namespace NYsonPull { - namespace NException { - class TBadStream: public std::exception { +namespace NYsonPull { + namespace NException { + class TBadStream: public std::exception { TString Message_; TPositionInfo Position_; mutable TString FormattedMessage_; - public: - TBadStream( + public: + TBadStream( TString message, - const TPositionInfo& position) + const TPositionInfo& position) : Message_(std::move(message)) , Position_(position) - { - } + { + } const TPositionInfo& Position() const { return Position_; - } + } - const char* what() const noexcept override; - }; + const char* what() const noexcept override; + }; - class TBadInput: public TBadStream { - public: - using TBadStream::TBadStream; - }; + class TBadInput: public TBadStream { + public: + using TBadStream::TBadStream; + }; - class TBadOutput: public TBadStream { - public: - using TBadStream::TBadStream; - }; + class TBadOutput: public TBadStream { + public: + using TBadStream::TBadStream; + }; - class TSystemError: public std::exception { + class TSystemError: public std::exception { int SavedErrno_; - public: - TSystemError(); - TSystemError(int saved_errno) + public: + TSystemError(); + TSystemError(int saved_errno) : SavedErrno_{saved_errno} { - } + } - int saved_errno() const noexcept { + int saved_errno() const noexcept { return SavedErrno_; - } + } - const char* what() const noexcept override; - }; - } -} + const char* what() const noexcept override; + }; + } +} diff --git a/library/cpp/yson_pull/input.h b/library/cpp/yson_pull/input.h index 2cdfae857e..97cad44987 100644 --- a/library/cpp/yson_pull/input.h +++ b/library/cpp/yson_pull/input.h @@ -13,69 +13,69 @@ class IInputStream; class IZeroCopyInput; -namespace NYsonPull { - namespace NInput { - //! \brief Input stream adaptor interface. - //! - //! Represents a model of a chunked input data stream. - class IStream { - input_buffer buffer_; - bool at_end_ = false; - - public: - virtual ~IStream() = default; - - bool at_end() const { - return at_end_; - } - - input_buffer& buffer() noexcept { - return buffer_; - } - const input_buffer& buffer() const noexcept { - return buffer_; - } - - void fill_buffer() { - while (buffer_.is_empty() && !at_end()) { - at_end_ = do_fill_buffer() == result::at_end; - } - } - - protected: - enum class result { - have_more_data, //! May continue reading - at_end, //! Reached end of stream - }; - - //! \brief Read next chunk of data. - //! - //! The implementation is to discard the buffer contents - //! and reset the buffer to a next chunk of data. - //! End-of-stream condition is to be reported via return value. - //! - //! Read is assumed to always succeed unless it throws an exception. - virtual result do_fill_buffer() = 0; - }; - - //! \brief Read data from a contiguous memory block (i.e. a string) - //! - //! Does not take ownership on memory. +namespace NYsonPull { + namespace NInput { + //! \brief Input stream adaptor interface. + //! + //! Represents a model of a chunked input data stream. + class IStream { + input_buffer buffer_; + bool at_end_ = false; + + public: + virtual ~IStream() = default; + + bool at_end() const { + return at_end_; + } + + input_buffer& buffer() noexcept { + return buffer_; + } + const input_buffer& buffer() const noexcept { + return buffer_; + } + + void fill_buffer() { + while (buffer_.is_empty() && !at_end()) { + at_end_ = do_fill_buffer() == result::at_end; + } + } + + protected: + enum class result { + have_more_data, //! May continue reading + at_end, //! Reached end of stream + }; + + //! \brief Read next chunk of data. + //! + //! The implementation is to discard the buffer contents + //! and reset the buffer to a next chunk of data. + //! End-of-stream condition is to be reported via return value. + //! + //! Read is assumed to always succeed unless it throws an exception. + virtual result do_fill_buffer() = 0; + }; + + //! \brief Read data from a contiguous memory block (i.e. a string) + //! + //! Does not take ownership on memory. THolder<IStream> FromMemory(TStringBuf data); - //! \brief Read data from C FILE* object. - //! - //! Does not take ownership on file object. - //! Data is buffered internally regardless of file buffering. + //! \brief Read data from C FILE* object. + //! + //! Does not take ownership on file object. + //! Data is buffered internally regardless of file buffering. THolder<IStream> FromStdioFile(FILE* file, size_t buffer_size = 65536); - //! \brief Read data from POSIX file descriptor. - //! - //! Does not take ownership on streambuf. + //! \brief Read data from POSIX file descriptor. + //! + //! Does not take ownership on streambuf. THolder<IStream> FromPosixFd(int fd, size_t buffer_size = 65536); THolder<IStream> FromZeroCopyInput(IZeroCopyInput* input); THolder<IStream> FromInputStream(IInputStream* input, size_t buffer_size = 65536); - } -} + } +} diff --git a/library/cpp/yson_pull/output.h b/library/cpp/yson_pull/output.h index 2d78107a93..fbe8a84ce7 100644 --- a/library/cpp/yson_pull/output.h +++ b/library/cpp/yson_pull/output.h @@ -15,42 +15,42 @@ //! \brief Output stream adaptor interface. //! //! Represents a model of an optionally-buffered writer. -namespace NYsonPull { - namespace NOutput { - class IStream { - output_buffer buffer_; +namespace NYsonPull { + namespace NOutput { + class IStream { + output_buffer buffer_; - public: - virtual ~IStream() = default; + public: + virtual ~IStream() = default; - output_buffer& buffer() noexcept { - return buffer_; - } - const output_buffer& buffer() const noexcept { - return buffer_; - } + output_buffer& buffer() noexcept { + return buffer_; + } + const output_buffer& buffer() const noexcept { + return buffer_; + } - void flush_buffer(TStringBuf extra = {}) { - if (!extra.empty() || !buffer_.is_empty()) { - do_flush_buffer(extra); - } - while (!buffer_.is_empty()) { - do_flush_buffer({}); - } - } + void flush_buffer(TStringBuf extra = {}) { + if (!extra.empty() || !buffer_.is_empty()) { + do_flush_buffer(extra); + } + while (!buffer_.is_empty()) { + do_flush_buffer({}); + } + } - protected: - //! \brief Flush data to underlying stream. - //! - //! The implementation is to flush the buffer contents AND - //! extra argument to underlying stream. - //! - //! This way, at zero buffer size this interface implements an unbuffered - //! stream (with an added cost of a virtual call per each write). - //! - //! Write is assumed to always succeed unless it throws an exception. - virtual void do_flush_buffer(TStringBuf extra) = 0; - }; + protected: + //! \brief Flush data to underlying stream. + //! + //! The implementation is to flush the buffer contents AND + //! extra argument to underlying stream. + //! + //! This way, at zero buffer size this interface implements an unbuffered + //! stream (with an added cost of a virtual call per each write). + //! + //! Write is assumed to always succeed unless it throws an exception. + virtual void do_flush_buffer(TStringBuf extra) = 0; + }; //! \brief Write data to C FILE* object. THolder<IStream> FromStdioFile(FILE* file, size_t buffer_size = 0); @@ -61,5 +61,5 @@ namespace NYsonPull { THolder<IStream> FromOutputStream(IOutputStream* output, size_t buffer_size = 65536); THolder<IStream> FromString(TString* output, size_t buffer_size = 1024); - } -} + } +} diff --git a/library/cpp/yson_pull/position_info.h b/library/cpp/yson_pull/position_info.h index a65c4663a9..3d6b86b409 100644 --- a/library/cpp/yson_pull/position_info.h +++ b/library/cpp/yson_pull/position_info.h @@ -4,20 +4,20 @@ #include <util/system/types.h> namespace NYsonPull { - struct TPositionInfo { + struct TPositionInfo { TMaybe<ui64> Offset; TMaybe<ui64> Line; TMaybe<ui64> Column; - TPositionInfo() = default; - TPositionInfo( - TMaybe<ui64> offset_, - TMaybe<ui64> line_ = Nothing(), - TMaybe<ui64> column_ = Nothing()) + TPositionInfo() = default; + TPositionInfo( + TMaybe<ui64> offset_, + TMaybe<ui64> line_ = Nothing(), + TMaybe<ui64> column_ = Nothing()) : Offset{offset_} , Line{line_} , Column{column_} { - } - }; + } + }; -} +} diff --git a/library/cpp/yson_pull/range.h b/library/cpp/yson_pull/range.h index f4fcf3f206..55194f8a89 100644 --- a/library/cpp/yson_pull/range.h +++ b/library/cpp/yson_pull/range.h @@ -6,30 +6,30 @@ namespace NYsonPull { class TStreamEventsRange: public TInputRangeAdaptor<TStreamEventsRange> { - TReader Reader_; - bool AtEnd; + TReader Reader_; + bool AtEnd; - public: + public: TStreamEventsRange(THolder<NInput::IStream> stream, EStreamType mode) - : Reader_{std::move(stream), mode} - , AtEnd(false) - { - } + : Reader_{std::move(stream), mode} + , AtEnd(false) + { + } const TEvent* Last() const noexcept { return &Reader_.LastEvent(); - } + } const TEvent* Next() { - if (Y_UNLIKELY(AtEnd)) { - return nullptr; - } + if (Y_UNLIKELY(AtEnd)) { + return nullptr; + } auto* event = &Reader_.NextEvent(); if (event->Type() == EEventType::EndStream) { - AtEnd = true; - } - return event; + AtEnd = true; + } + return event; } - }; -} + }; +} diff --git a/library/cpp/yson_pull/read_ops.cpp b/library/cpp/yson_pull/read_ops.cpp index 9d7e6a4a2d..d565ac7e51 100644 --- a/library/cpp/yson_pull/read_ops.cpp +++ b/library/cpp/yson_pull/read_ops.cpp @@ -7,53 +7,53 @@ namespace { bool TrySkipValueUntil(EEventType end, TReader& reader) { const auto& event = reader.NextEvent(); if (event.Type() == end) { - return false; - } - SkipCurrentValue(event, reader); - return true; - } + return false; + } + SkipCurrentValue(event, reader); + return true; + } bool TrySkipKeyValueUntil(EEventType end, TReader& reader) { const auto& event = reader.NextEvent(); if (event.Type() == end) { - return false; - } + return false; + } Expect(event, EEventType::Key); - SkipValue(reader); - return true; - } + SkipValue(reader); + return true; + } } void NYsonPull::NReadOps::SkipCurrentValue(const TEvent& event, TReader& reader) { switch (event.Type()) { case EEventType::BeginList: while (TrySkipValueUntil(EEventType::EndList, reader)) { - } - return; + } + return; case EEventType::BeginMap: while (TrySkipKeyValueUntil(EEventType::EndMap, reader)) { - } - return; + } + return; case EEventType::BeginAttributes: while (TrySkipKeyValueUntil(EEventType::EndAttributes, reader)) { - } - // attributes after attributes are disallowed in TReader - SkipValue(reader); - return; + } + // attributes after attributes are disallowed in TReader + SkipValue(reader); + return; case EEventType::Scalar: - return; + return; - default: - throw yexception() << "Unexpected event: " << event; - } + default: + throw yexception() << "Unexpected event: " << event; + } } void NYsonPull::NReadOps::SkipValue(TReader& reader) { const auto& event = reader.NextEvent(); - SkipCurrentValue(event, reader); + SkipCurrentValue(event, reader); } void NYsonPull::NReadOps::SkipControlRecords(TReader& reader) { diff --git a/library/cpp/yson_pull/read_ops.h b/library/cpp/yson_pull/read_ops.h index 5c084983ea..51e619c3c6 100644 --- a/library/cpp/yson_pull/read_ops.h +++ b/library/cpp/yson_pull/read_ops.h @@ -20,22 +20,22 @@ * */ -namespace NYsonPull { - namespace NReadOps { - class TExpectationFailure: public TWithBackTrace<yexception> { +namespace NYsonPull { + namespace NReadOps { + class TExpectationFailure: public TWithBackTrace<yexception> { }; - inline void Expect(const TEvent& got, EEventType expected) { + inline void Expect(const TEvent& got, EEventType expected) { Y_ENSURE_EX( got.Type() == expected, TExpectationFailure() << "expected " << expected << ", got " << got); - } + } - inline void Expect(const TScalar& got, EScalarType expected) { + inline void Expect(const TScalar& got, EScalarType expected) { Y_ENSURE_EX( got.Type() == expected, TExpectationFailure() << "expected scalar " << expected << ", got " << got); - } + } // ExpectBegin{X} functions verify that last event WAS X // SkipBegin{X} functions verify that next event WILL BE X and CONSUME it @@ -48,95 +48,95 @@ namespace NYsonPull { Expect(reader.NextEvent(), EEventType::BeginStream); } - inline void ExpectBeginMap(TReader& reader) { + inline void ExpectBeginMap(TReader& reader) { Expect(reader.LastEvent(), EEventType::BeginMap); - } + } - inline void SkipBeginMap(TReader& reader) { + inline void SkipBeginMap(TReader& reader) { Expect(reader.NextEvent(), EEventType::BeginMap); - } + } - inline void ExpectBeginList(TReader& reader) { + inline void ExpectBeginList(TReader& reader) { Expect(reader.LastEvent(), EEventType::BeginList); - } + } - inline void SkipBeginList(TReader& reader) { + inline void SkipBeginList(TReader& reader) { Expect(reader.NextEvent(), EEventType::BeginList); - } + } - inline bool ReadListItem(TReader& reader) { + inline bool ReadListItem(TReader& reader) { return reader.NextEvent().Type() != EEventType::EndList; - } + } - inline TMaybe<TStringBuf> ReadKey(TReader& reader) { + inline TMaybe<TStringBuf> ReadKey(TReader& reader) { const auto& event = reader.NextEvent(); switch (event.Type()) { case EEventType::Key: return event.AsString(); case EEventType::EndMap: - return Nothing(); - default: - ythrow yexception() << "Unexpected event: " << event; - } - } + return Nothing(); + default: + ythrow yexception() << "Unexpected event: " << event; + } + } - template <typename T = const TScalar&> - inline T ReadScalar(TReader& reader); + template <typename T = const TScalar&> + inline T ReadScalar(TReader& reader); - template <> - inline const TScalar& ReadScalar<const TScalar&>(TReader& reader) { + template <> + inline const TScalar& ReadScalar<const TScalar&>(TReader& reader) { const auto& event = reader.NextEvent(); Expect(event, EEventType::Scalar); return event.AsScalar(); - } + } - template <> - inline i64 ReadScalar<i64>(TReader& reader) { - const auto& scalar = ReadScalar(reader); + template <> + inline i64 ReadScalar<i64>(TReader& reader) { + const auto& scalar = ReadScalar(reader); Expect(scalar, EScalarType::Int64); return scalar.AsInt64(); - } + } - template <> - inline ui64 ReadScalar<ui64>(TReader& reader) { - const auto& scalar = ReadScalar(reader); + template <> + inline ui64 ReadScalar<ui64>(TReader& reader) { + const auto& scalar = ReadScalar(reader); Expect(scalar, EScalarType::UInt64); return scalar.AsUInt64(); - } + } - template <> - inline double ReadScalar<double>(TReader& reader) { - const auto& scalar = ReadScalar(reader); + template <> + inline double ReadScalar<double>(TReader& reader) { + const auto& scalar = ReadScalar(reader); Expect(scalar, EScalarType::Float64); return scalar.AsFloat64(); - } + } - template <> - inline TStringBuf ReadScalar<TStringBuf>(TReader& reader) { - const auto& scalar = ReadScalar(reader); + template <> + inline TStringBuf ReadScalar<TStringBuf>(TReader& reader) { + const auto& scalar = ReadScalar(reader); Expect(scalar, EScalarType::String); return scalar.AsString(); - } + } - template <> + template <> inline TString ReadScalar<TString>(TReader& reader) { return TString(ReadScalar<TStringBuf>(reader)); - } + } - template <> - inline bool ReadScalar<bool>(TReader& reader) { - const auto& scalar = ReadScalar(reader); + template <> + inline bool ReadScalar<bool>(TReader& reader) { + const auto& scalar = ReadScalar(reader); Expect(scalar, EScalarType::Boolean); return scalar.AsBoolean(); - } + } - // Skip value that was already started with `event` - void SkipCurrentValue(const TEvent& event, TReader& reader); + // Skip value that was already started with `event` + void SkipCurrentValue(const TEvent& event, TReader& reader); - // Skip value that starts at `reader.next_event()` - void SkipValue(TReader& reader); + // Skip value that starts at `reader.next_event()` + void SkipValue(TReader& reader); // Skip values with attributes, wait for map value void SkipControlRecords(TReader& reader); - } -} + } +} diff --git a/library/cpp/yson_pull/reader.cpp b/library/cpp/yson_pull/reader.cpp index ea26852756..111c8dfd0c 100644 --- a/library/cpp/yson_pull/reader.cpp +++ b/library/cpp/yson_pull/reader.cpp @@ -5,18 +5,18 @@ using namespace NYsonPull; TReader::TReader( THolder<NInput::IStream> stream, - EStreamType mode) + EStreamType mode) : Stream_{std::move(stream)} , Impl_{MakeHolder<NDetail::reader_impl>(*Stream_, mode)} { -} +} -TReader::TReader(TReader&& other) noexcept +TReader::TReader(TReader&& other) noexcept : Stream_{std::move(other.Stream_)} , Impl_{std::move(other.Impl_)} { -} +} -TReader::~TReader() { -} +TReader::~TReader() { +} const TEvent& TReader::NextEvent() { return Impl_->next_event(); diff --git a/library/cpp/yson_pull/reader.h b/library/cpp/yson_pull/reader.h index f839b19071..6326d6eba3 100644 --- a/library/cpp/yson_pull/reader.h +++ b/library/cpp/yson_pull/reader.h @@ -9,29 +9,29 @@ #include <memory> namespace NYsonPull { - namespace NDetail { - class reader_impl; - } - - //! \brief YSON reader facade class. - //! - //! Owns an input stream. - class TReader { + namespace NDetail { + class reader_impl; + } + + //! \brief YSON reader facade class. + //! + //! Owns an input stream. + class TReader { THolder<NInput::IStream> Stream_; THolder<NDetail::reader_impl> Impl_; - public: - TReader(THolder<NInput::IStream> stream, EStreamType mode); - TReader(TReader&&) noexcept; - ~TReader(); + public: + TReader(THolder<NInput::IStream> stream, EStreamType mode); + TReader(TReader&&) noexcept; + ~TReader(); - //! \brief Advance stream to next event and return it. - //! + //! \brief Advance stream to next event and return it. + //! //! Any event data is invalidated by a call to NextEvent(); const TEvent& NextEvent(); - //! \brief Get last returned event. + //! \brief Get last returned event. const TEvent& LastEvent() const noexcept; - }; + }; -} +} diff --git a/library/cpp/yson_pull/scalar.cpp b/library/cpp/yson_pull/scalar.cpp index 4325542e7a..789ae5c375 100644 --- a/library/cpp/yson_pull/scalar.cpp +++ b/library/cpp/yson_pull/scalar.cpp @@ -6,7 +6,7 @@ using namespace NYsonPull; -template <> +template <> void Out<TScalar>(IOutputStream& out, const TScalar& value) { out << '(' << value.Type(); if (value.Type() != EScalarType::Entity) { @@ -28,13 +28,13 @@ void Out<TScalar>(IOutputStream& out, const TScalar& value) { case EScalarType::Float64: out << value.AsFloat64(); break; - default: - break; + default: + break; } out << ')'; } -bool NYsonPull::operator==(const TScalar& left, const TScalar& right) noexcept { +bool NYsonPull::operator==(const TScalar& left, const TScalar& right) noexcept { if (left.Type() != right.Type()) { return false; } @@ -51,7 +51,7 @@ bool NYsonPull::operator==(const TScalar& left, const TScalar& right) noexcept { return left.AsFloat64() == right.AsFloat64(); case EScalarType::Entity: return true; - default: + default: Y_UNREACHABLE(); } } diff --git a/library/cpp/yson_pull/scalar.h b/library/cpp/yson_pull/scalar.h index 509fce8b5e..a105e36de2 100644 --- a/library/cpp/yson_pull/scalar.h +++ b/library/cpp/yson_pull/scalar.h @@ -7,24 +7,24 @@ #include <util/system/yassert.h> namespace NYsonPull { - //! \brief YSON TScalar value type tag - enum class EScalarType { + //! \brief YSON TScalar value type tag + enum class EScalarType { Entity = YSON_SCALAR_ENTITY, Boolean = YSON_SCALAR_BOOLEAN, Int64 = YSON_SCALAR_INT64, UInt64 = YSON_SCALAR_UINT64, Float64 = YSON_SCALAR_FLOAT64, String = YSON_SCALAR_STRING, - }; + }; - //! \brief YSON TScalar value variant - class TScalar { - //! \internal \brief YSON TScalar value underlying representation + //! \brief YSON TScalar value variant + class TScalar { + //! \internal \brief YSON TScalar value underlying representation union TScalarValue { struct TScalarStringRef { const char* Data; size_t Size; - }; + }; ui8 AsNothing[1]; bool AsBoolean; @@ -35,76 +35,76 @@ namespace NYsonPull { constexpr TScalarValue() : AsNothing{} { - } + } explicit constexpr TScalarValue(bool value) : AsBoolean{value} { - } + } explicit constexpr TScalarValue(i64 value) : AsInt64{value} { - } + } explicit constexpr TScalarValue(ui64 value) : AsUInt64{value} { - } + } explicit constexpr TScalarValue(double value) : AsFloat64{value} { - } + } explicit constexpr TScalarValue(TStringBuf value) : AsString{value.data(), value.size()} { - } - }; - static_assert( + } + }; + static_assert( sizeof(TScalarValue) == sizeof(TStringBuf), - "bad scalar_value size"); + "bad scalar_value size"); EScalarType Type_; TScalarValue Value_; - public: - constexpr TScalar() + public: + constexpr TScalar() : Type_{EScalarType::Entity} { - } + } - explicit constexpr TScalar(bool value) + explicit constexpr TScalar(bool value) : Type_{EScalarType::Boolean} , Value_{value} { - } + } - explicit constexpr TScalar(i64 value) + explicit constexpr TScalar(i64 value) : Type_{EScalarType::Int64} , Value_{value} { - } + } - explicit constexpr TScalar(ui64 value) + explicit constexpr TScalar(ui64 value) : Type_{EScalarType::UInt64} , Value_{value} { - } + } - explicit constexpr TScalar(double value) + explicit constexpr TScalar(double value) : Type_{EScalarType::Float64} , Value_{value} { - } + } - explicit constexpr TScalar(TStringBuf value) + explicit constexpr TScalar(TStringBuf value) : Type_{EScalarType::String} , Value_{value} { - } + } - // Disambiguation for literal constants - // In the absence of this constructor, - // they get implicitly converted to bool (yikes!) - explicit TScalar(const char* value) + // Disambiguation for literal constants + // In the absence of this constructor, + // they get implicitly converted to bool (yikes!) + explicit TScalar(const char* value) : Type_{EScalarType::String} , Value_{TStringBuf{value}} { - } + } EScalarType Type() const { return Type_; - } + } #define CAST_TO(Type) \ Y_ASSERT(Type_ == EScalarType::Type); \ @@ -112,35 +112,35 @@ namespace NYsonPull { bool AsBoolean() const { CAST_TO(Boolean); - } + } i64 AsInt64() const { CAST_TO(Int64); - } + } ui64 AsUInt64() const { CAST_TO(UInt64); - } + } double AsFloat64() const { CAST_TO(Float64); - } + } #undef CAST_TO TStringBuf AsString() const { Y_ASSERT(Type_ == EScalarType::String); - return TStringBuf{ + return TStringBuf{ Value_.AsString.Data, Value_.AsString.Size, - }; - } + }; + } const TScalarValue& AsUnsafeValue() const { return Value_; - } - }; + } + }; - bool operator==(const TScalar& left, const TScalar& right) noexcept; + bool operator==(const TScalar& left, const TScalar& right) noexcept; - inline bool operator!=(const TScalar& left, const TScalar& right) noexcept { - return !(left == right); - } + inline bool operator!=(const TScalar& left, const TScalar& right) noexcept { + return !(left == right); + } -} +} diff --git a/library/cpp/yson_pull/stream_type.h b/library/cpp/yson_pull/stream_type.h index beac87fe1b..293d166ac1 100644 --- a/library/cpp/yson_pull/stream_type.h +++ b/library/cpp/yson_pull/stream_type.h @@ -3,9 +3,9 @@ #include "cyson_enums.h" namespace NYsonPull { - enum class EStreamType { - Node = YSON_STREAM_TYPE_NODE, - ListFragment = YSON_STREAM_TYPE_LIST_FRAGMENT, - MapFragment = YSON_STREAM_TYPE_MAP_FRAGMENT, - }; -} + enum class EStreamType { + Node = YSON_STREAM_TYPE_NODE, + ListFragment = YSON_STREAM_TYPE_LIST_FRAGMENT, + MapFragment = YSON_STREAM_TYPE_MAP_FRAGMENT, + }; +} diff --git a/library/cpp/yson_pull/ut/cescape_ut.cpp b/library/cpp/yson_pull/ut/cescape_ut.cpp index 6628ba1d15..a1bb07d2fe 100644 --- a/library/cpp/yson_pull/ut/cescape_ut.cpp +++ b/library/cpp/yson_pull/ut/cescape_ut.cpp @@ -6,66 +6,66 @@ using namespace NYsonPull::NDetail; namespace { void test_roundtrip(const TVector<ui8>& str) { - TStringBuf str_buf( - reinterpret_cast<const char*>(str.data()), - str.size()); - auto tmp = NCEscape::encode(str_buf); - auto dest = NCEscape::decode(tmp); - UNIT_ASSERT_VALUES_EQUAL_C( - str_buf, TStringBuf(dest), - "A[" << str.size() << "]: " << str_buf << '\n' - << "B[" << tmp.size() << "]: " << tmp << '\n' - << "C[" << dest.size() << "]: " << dest); - } + TStringBuf str_buf( + reinterpret_cast<const char*>(str.data()), + str.size()); + auto tmp = NCEscape::encode(str_buf); + auto dest = NCEscape::decode(tmp); + UNIT_ASSERT_VALUES_EQUAL_C( + str_buf, TStringBuf(dest), + "A[" << str.size() << "]: " << str_buf << '\n' + << "B[" << tmp.size() << "]: " << tmp << '\n' + << "C[" << dest.size() << "]: " << dest); + } - template <size_t N> + template <size_t N> void test_exhaustive(TVector<ui8>& str) { - for (int i = 0; i < 256; ++i) { - str[str.size() - N] = static_cast<char>(i); - test_exhaustive<N - 1>(str); - } - } + for (int i = 0; i < 256; ++i) { + str[str.size() - N] = static_cast<char>(i); + test_exhaustive<N - 1>(str); + } + } - template <> + template <> void test_exhaustive<0>(TVector<ui8>& str) { - test_roundtrip(str); + test_roundtrip(str); } - template <size_t N> - void test_exhaustive() { + template <size_t N> + void test_exhaustive() { TVector<ui8> str(N, ' '); - test_exhaustive<N>(str); - } + test_exhaustive<N>(str); + } } // anonymous namespace Y_UNIT_TEST_SUITE(CEscape) { Y_UNIT_TEST(ExhaustiveOneChar) { - test_exhaustive<1>(); - } + test_exhaustive<1>(); + } Y_UNIT_TEST(ExhaustiveTwoChars) { - test_exhaustive<2>(); - } + test_exhaustive<2>(); + } Y_UNIT_TEST(ExhaustiveThreeChars) { - test_exhaustive<3>(); - } + test_exhaustive<3>(); + } Y_UNIT_TEST(SpecialEscapeEncode) { - //UNIT_ASSERT_VALUES_EQUAL(R"(\b)", NCEscape::encode("\b")); - //UNIT_ASSERT_VALUES_EQUAL(R"(\f)", NCEscape::encode("\f")); - UNIT_ASSERT_VALUES_EQUAL(R"(\n)", NCEscape::encode("\n")); - UNIT_ASSERT_VALUES_EQUAL(R"(\r)", NCEscape::encode("\r")); - UNIT_ASSERT_VALUES_EQUAL(R"(\t)", NCEscape::encode("\t")); - } + //UNIT_ASSERT_VALUES_EQUAL(R"(\b)", NCEscape::encode("\b")); + //UNIT_ASSERT_VALUES_EQUAL(R"(\f)", NCEscape::encode("\f")); + UNIT_ASSERT_VALUES_EQUAL(R"(\n)", NCEscape::encode("\n")); + UNIT_ASSERT_VALUES_EQUAL(R"(\r)", NCEscape::encode("\r")); + UNIT_ASSERT_VALUES_EQUAL(R"(\t)", NCEscape::encode("\t")); + } Y_UNIT_TEST(SpecialEscapeDecode) { - UNIT_ASSERT_VALUES_EQUAL("\b", NCEscape::decode(R"(\b)")); - UNIT_ASSERT_VALUES_EQUAL("\f", NCEscape::decode(R"(\f)")); - UNIT_ASSERT_VALUES_EQUAL("\n", NCEscape::decode(R"(\n)")); - UNIT_ASSERT_VALUES_EQUAL("\r", NCEscape::decode(R"(\r)")); - UNIT_ASSERT_VALUES_EQUAL("\t", NCEscape::decode(R"(\t)")); - } + UNIT_ASSERT_VALUES_EQUAL("\b", NCEscape::decode(R"(\b)")); + UNIT_ASSERT_VALUES_EQUAL("\f", NCEscape::decode(R"(\f)")); + UNIT_ASSERT_VALUES_EQUAL("\n", NCEscape::decode(R"(\n)")); + UNIT_ASSERT_VALUES_EQUAL("\r", NCEscape::decode(R"(\r)")); + UNIT_ASSERT_VALUES_EQUAL("\t", NCEscape::decode(R"(\t)")); + } } // Y_UNIT_TEST_SUITE(CEscape) diff --git a/library/cpp/yson_pull/ut/loop_ut.cpp b/library/cpp/yson_pull/ut/loop_ut.cpp index 8c7b11dd1c..706a00981d 100644 --- a/library/cpp/yson_pull/ut/loop_ut.cpp +++ b/library/cpp/yson_pull/ut/loop_ut.cpp @@ -14,26 +14,26 @@ #endif namespace { - constexpr const char* alphabet = - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - - void generate(NYsonPull::TWriter& writer, size_t count) { + constexpr const char* alphabet = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + void generate(NYsonPull::TWriter& writer, size_t count) { writer.BeginStream(); - for (size_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { writer.BeginMap() .Key("ints") .BeginList() @@ -81,40 +81,40 @@ namespace { .String(alphabet) .EndList() .EndMap(); - } + } writer.EndStream(); } #ifdef __clang__ - // XXX: With all the macros below (esp. UNIT_ASSERT_VALUES_EQUAL) unfolded, - // the time it takes clang to optimize generated code becomes abysmal. - // Locally disabling optimization brings it back to normal. - __attribute__((optnone)) + // XXX: With all the macros below (esp. UNIT_ASSERT_VALUES_EQUAL) unfolded, + // the time it takes clang to optimize generated code becomes abysmal. + // Locally disabling optimization brings it back to normal. + __attribute__((optnone)) #endif // __clang__ - void - verify(NYsonPull::TReader& reader, size_t count) { -#define NEXT(name__) \ - { \ + void + verify(NYsonPull::TReader& reader, size_t count) { +#define NEXT(name__) \ + { \ auto& name__ = reader.NextEvent(); // SCOPED_TRACE(e); #define END_NEXT } -#define NEXT_TYPE(type__) \ - NEXT(e) { \ +#define NEXT_TYPE(type__) \ + NEXT(e) { \ UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::type__, e.Type()); \ - } \ - END_NEXT -#define NEXT_KEY(key__) \ - NEXT(e) { \ + } \ + END_NEXT +#define NEXT_KEY(key__) \ + NEXT(e) { \ UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); \ UNIT_ASSERT_VALUES_EQUAL(key__, e.AsString()); \ - } \ - END_NEXT + } \ + END_NEXT #define NEXT_SCALAR(type__, value__) \ NEXT(e) { \ UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); \ UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EScalarType::type__, e.AsScalar().Type()); \ UNIT_ASSERT_VALUES_EQUAL(value__, e.AsScalar().As##type__()); \ } \ - END_NEXT + END_NEXT #define NEXT_ENTITY() \ NEXT(e) { \ UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); \ @@ -122,65 +122,65 @@ namespace { } \ END_NEXT #define NEXT_FLOAT64(value__) \ - NEXT(e) { \ + NEXT(e) { \ UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); \ UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EScalarType::Float64, e.AsScalar().Type()); \ UNIT_ASSERT_DOUBLES_EQUAL(value__, e.AsScalar().AsFloat64(), 1e-5); \ - } \ - END_NEXT + } \ + END_NEXT - constexpr auto true_ = true; - constexpr auto false_ = false; + constexpr auto true_ = true; + constexpr auto false_ = false; NEXT_TYPE(BeginStream); - for (size_t i = 0; i < count; ++i) { + for (size_t i = 0; i < count; ++i) { NEXT_TYPE(BeginMap); - NEXT_KEY("ints") { + NEXT_KEY("ints") { NEXT_TYPE(BeginList); NEXT_SCALAR(Int64, 0); NEXT_SCALAR(Int64, -1); NEXT_SCALAR(Int64, 1000); NEXT_SCALAR(Int64, -1000); NEXT_TYPE(EndList); - } - NEXT_KEY("uints") { + } + NEXT_KEY("uints") { NEXT_TYPE(BeginList); NEXT_SCALAR(UInt64, 0U); NEXT_SCALAR(UInt64, 1000U); NEXT_SCALAR(UInt64, 10000000U); NEXT_TYPE(EndList); - } - NEXT_KEY("entities") { + } + NEXT_KEY("entities") { NEXT_TYPE(BeginList); - NEXT_ENTITY(); + NEXT_ENTITY(); NEXT_TYPE(BeginAttributes) { - NEXT_KEY("color") { + NEXT_KEY("color") { NEXT_SCALAR(String, "blue"); - } - NEXT_KEY("size") { + } + NEXT_KEY("size") { NEXT_SCALAR(Int64, 100); - } + } } NEXT_TYPE(EndAttributes); - NEXT_ENTITY(); - NEXT_ENTITY(); + NEXT_ENTITY(); + NEXT_ENTITY(); NEXT_TYPE(EndList); - } - NEXT_KEY("booleans") { + } + NEXT_KEY("booleans") { NEXT_TYPE(BeginList); NEXT_SCALAR(Boolean, true_); NEXT_SCALAR(Boolean, false_); NEXT_SCALAR(Boolean, true_); NEXT_TYPE(EndList); - } - NEXT_KEY("floats") { + } + NEXT_KEY("floats") { NEXT_TYPE(BeginList); - NEXT_FLOAT64(0.0); - NEXT_FLOAT64(13.0e30); - NEXT_FLOAT64(M_PI); + NEXT_FLOAT64(0.0); + NEXT_FLOAT64(13.0e30); + NEXT_FLOAT64(M_PI); NEXT_TYPE(EndList); - } - NEXT_KEY("strings") { + } + NEXT_KEY("strings") { NEXT_TYPE(BeginList); NEXT_SCALAR(String, "hello"); NEXT_SCALAR(String, ""); @@ -188,7 +188,7 @@ namespace { NEXT_SCALAR(String, "oh\nwow"); NEXT_SCALAR(String, alphabet); NEXT_TYPE(EndList); - } + } NEXT_TYPE(EndMap); } NEXT_TYPE(EndStream); @@ -198,14 +198,14 @@ namespace { #undef NEXT_TYPE #undef NEXT_KEY #undef NEXT_SCALAR - } + } - class sys_error {}; + class sys_error {}; IOutputStream& operator<<(IOutputStream& stream, const sys_error&) { - stream << strerror(errno); - return stream; - } + stream << strerror(errno); + return stream; + } NYsonPull::TReader make_reader(THolder<NYsonPull::NInput::IStream> stream) { return NYsonPull::TReader( @@ -232,151 +232,151 @@ namespace { } #ifdef _unix_ - template <typename Here, typename There> - void pipe(Here&& reader, There&& writer) { - int fildes[2]; - UNIT_ASSERT_VALUES_EQUAL_C(0, ::pipe(fildes), sys_error()); - auto read_fd = fildes[0]; - auto write_fd = fildes[1]; - - auto pid = ::fork(); - UNIT_ASSERT_C(pid >= 0, sys_error()); - if (pid > 0) { - // parent - UNIT_ASSERT_VALUES_EQUAL_C(0, ::close(write_fd), sys_error()); - reader(read_fd); - UNIT_ASSERT_VALUES_EQUAL_C(0, ::close(read_fd), sys_error()); - } else { - // child - UNIT_ASSERT_VALUES_EQUAL_C(0, ::close(read_fd), sys_error()); - UNIT_ASSERT_NO_EXCEPTION(writer(write_fd)); - UNIT_ASSERT_VALUES_EQUAL_C(0, ::close(write_fd), sys_error()); - ::exit(0); - } - int stat_loc; - UNIT_ASSERT_VALUES_EQUAL_C(pid, ::waitpid(pid, &stat_loc, 0), sys_error()); + template <typename Here, typename There> + void pipe(Here&& reader, There&& writer) { + int fildes[2]; + UNIT_ASSERT_VALUES_EQUAL_C(0, ::pipe(fildes), sys_error()); + auto read_fd = fildes[0]; + auto write_fd = fildes[1]; + + auto pid = ::fork(); + UNIT_ASSERT_C(pid >= 0, sys_error()); + if (pid > 0) { + // parent + UNIT_ASSERT_VALUES_EQUAL_C(0, ::close(write_fd), sys_error()); + reader(read_fd); + UNIT_ASSERT_VALUES_EQUAL_C(0, ::close(read_fd), sys_error()); + } else { + // child + UNIT_ASSERT_VALUES_EQUAL_C(0, ::close(read_fd), sys_error()); + UNIT_ASSERT_NO_EXCEPTION(writer(write_fd)); + UNIT_ASSERT_VALUES_EQUAL_C(0, ::close(write_fd), sys_error()); + ::exit(0); + } + int stat_loc; + UNIT_ASSERT_VALUES_EQUAL_C(pid, ::waitpid(pid, &stat_loc, 0), sys_error()); } - template <typename Function> - void test_posix_fd( - Function make_writer, - size_t nrepeat, - size_t read_buffer_size, - size_t write_buffer_size) { - pipe( - [&](int fd) { + template <typename Function> + void test_posix_fd( + Function make_writer, + size_t nrepeat, + size_t read_buffer_size, + size_t write_buffer_size) { + pipe( + [&](int fd) { auto reader = make_reader(NYsonPull::NInput::FromPosixFd(fd, read_buffer_size)); - verify(reader, nrepeat); - }, - [&](int fd) { + verify(reader, nrepeat); + }, + [&](int fd) { auto writer = make_writer(NYsonPull::NOutput::FromPosixFd(fd, write_buffer_size)); - generate(writer, nrepeat); - }); - } - - template <typename Function> - void test_stdio_file( - Function make_writer, - size_t nrepeat, - size_t read_buffer_size, - size_t write_buffer_size) { - pipe( - [&](int fd) { - auto file = ::fdopen(fd, "rb"); - UNIT_ASSERT_C(file != nullptr, sys_error()); + generate(writer, nrepeat); + }); + } + + template <typename Function> + void test_stdio_file( + Function make_writer, + size_t nrepeat, + size_t read_buffer_size, + size_t write_buffer_size) { + pipe( + [&](int fd) { + auto file = ::fdopen(fd, "rb"); + UNIT_ASSERT_C(file != nullptr, sys_error()); auto reader = make_reader(NYsonPull::NInput::FromStdioFile(file, read_buffer_size)); - verify(reader, nrepeat); - }, - [&](int fd) { - auto file = ::fdopen(fd, "wb"); - Y_UNUSED(write_buffer_size); + verify(reader, nrepeat); + }, + [&](int fd) { + auto file = ::fdopen(fd, "wb"); + Y_UNUSED(write_buffer_size); auto writer = make_writer(NYsonPull::NOutput::FromStdioFile(file, write_buffer_size)); - generate(writer, nrepeat); - fflush(file); - }); - } + generate(writer, nrepeat); + fflush(file); + }); + } #endif - NYsonPull::TWriter text(THolder<NYsonPull::NOutput::IStream> stream) { + NYsonPull::TWriter text(THolder<NYsonPull::NOutput::IStream> stream) { return NYsonPull::MakeTextWriter( - std::move(stream), - NYsonPull::EStreamType::ListFragment); - } + std::move(stream), + NYsonPull::EStreamType::ListFragment); + } - NYsonPull::TWriter pretty_text(THolder<NYsonPull::NOutput::IStream> stream) { + NYsonPull::TWriter pretty_text(THolder<NYsonPull::NOutput::IStream> stream) { return NYsonPull::MakePrettyTextWriter( - std::move(stream), - NYsonPull::EStreamType::ListFragment); - } + std::move(stream), + NYsonPull::EStreamType::ListFragment); + } - NYsonPull::TWriter binary(THolder<NYsonPull::NOutput::IStream> stream) { + NYsonPull::TWriter binary(THolder<NYsonPull::NOutput::IStream> stream) { return NYsonPull::MakeBinaryWriter( - std::move(stream), - NYsonPull::EStreamType::ListFragment); - } - + std::move(stream), + NYsonPull::EStreamType::ListFragment); + } + } // anonymous namespace Y_UNIT_TEST_SUITE(Loop) { Y_UNIT_TEST(memory_pretty_text) { - test_memory(pretty_text, 100); - } + test_memory(pretty_text, 100); + } Y_UNIT_TEST(memory_text) { - test_memory(text, 100); - } + test_memory(text, 100); + } Y_UNIT_TEST(memory_binary) { - test_memory(binary, 100); - } + test_memory(binary, 100); + } #ifdef _unix_ Y_UNIT_TEST(posix_fd_pretty_text_buffered) { - test_posix_fd(pretty_text, 100, 1024, 1024); - } + test_posix_fd(pretty_text, 100, 1024, 1024); + } Y_UNIT_TEST(posix_fd_pretty_text_unbuffered) { - test_posix_fd(pretty_text, 100, 1, 0); - } + test_posix_fd(pretty_text, 100, 1, 0); + } Y_UNIT_TEST(posix_fd_text_buffered) { - test_posix_fd(text, 100, 1024, 1024); - } + test_posix_fd(text, 100, 1024, 1024); + } Y_UNIT_TEST(posix_fd_text_unbuffered) { - test_posix_fd(text, 100, 1, 0); - } + test_posix_fd(text, 100, 1, 0); + } Y_UNIT_TEST(posix_fd_binary_buffered) { - test_posix_fd(binary, 100, 1024, 1024); - } + test_posix_fd(binary, 100, 1024, 1024); + } Y_UNIT_TEST(posix_fd_binary_unbuffered) { - test_posix_fd(binary, 100, 1, 0); - } + test_posix_fd(binary, 100, 1, 0); + } Y_UNIT_TEST(stdio_file_pretty_text_buffered) { - test_stdio_file(pretty_text, 100, 1024, 1024); - } + test_stdio_file(pretty_text, 100, 1024, 1024); + } Y_UNIT_TEST(stdio_file_pretty_text_unbuffered) { - test_stdio_file(pretty_text, 100, 1, 0); - } + test_stdio_file(pretty_text, 100, 1, 0); + } Y_UNIT_TEST(stdio_file_text_buffered) { - test_stdio_file(text, 100, 1024, 1024); - } + test_stdio_file(text, 100, 1024, 1024); + } Y_UNIT_TEST(stdio_file_text_unbuffered) { - test_stdio_file(text, 100, 1, 0); - } + test_stdio_file(text, 100, 1, 0); + } Y_UNIT_TEST(stdio_file_binary_buffered) { - test_stdio_file(binary, 100, 1024, 1024); - } + test_stdio_file(binary, 100, 1024, 1024); + } Y_UNIT_TEST(stdio_file_binary_unbuffered) { - test_stdio_file(binary, 100, 1, 0); - } + test_stdio_file(binary, 100, 1, 0); + } #endif } // Y_UNIT_TEST_SUITE(Loop) diff --git a/library/cpp/yson_pull/ut/reader_ut.cpp b/library/cpp/yson_pull/ut/reader_ut.cpp index 1184265ddb..f208a94be1 100644 --- a/library/cpp/yson_pull/ut/reader_ut.cpp +++ b/library/cpp/yson_pull/ut/reader_ut.cpp @@ -7,19 +7,19 @@ #include <library/cpp/testing/unittest/registar.h> namespace { - NYsonPull::TReader memory_reader(TStringBuf data, NYsonPull::EStreamType mode) { - return NYsonPull::TReader( - NYsonPull::NInput::FromMemory(data), - mode); - } - - template <typename T> - void expect_scalar(const NYsonPull::TScalar& scalar, T value) { - UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{value}, scalar); - } - - template <> - void expect_scalar(const NYsonPull::TScalar& scalar, double value) { + NYsonPull::TReader memory_reader(TStringBuf data, NYsonPull::EStreamType mode) { + return NYsonPull::TReader( + NYsonPull::NInput::FromMemory(data), + mode); + } + + template <typename T> + void expect_scalar(const NYsonPull::TScalar& scalar, T value) { + UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{value}, scalar); + } + + template <> + void expect_scalar(const NYsonPull::TScalar& scalar, double value) { UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EScalarType::Float64, scalar.Type()); auto scalarValue = scalar.AsFloat64(); @@ -35,34 +35,34 @@ namespace { } else { UNIT_ASSERT_C(std::isinf(scalarValue) && (scalarValue < 0), message); } - } + } - template <typename T> - void test_scalar(TStringBuf data, T value) { - // SCOPED_TRACE(NYsonPull::detail::cescape::quote(data)); - auto reader = memory_reader(data, NYsonPull::EStreamType::Node); + template <typename T> + void test_scalar(TStringBuf data, T value) { + // SCOPED_TRACE(NYsonPull::detail::cescape::quote(data)); + auto reader = memory_reader(data, NYsonPull::EStreamType::Node); - try { + try { UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); - { + { auto& event = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, event.Type()); expect_scalar(event.AsScalar(), value); - } + } UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); - } catch (const std::exception& err) { - UNIT_FAIL(err.what()); + } catch (const std::exception& err) { + UNIT_FAIL(err.what()); } } - void consume(TStringBuf data, NYsonPull::EStreamType mode = NYsonPull::EStreamType::Node) { - // SCOPED_TRACE(NYsonPull::detail::cescape::quote(data)); + void consume(TStringBuf data, NYsonPull::EStreamType mode = NYsonPull::EStreamType::Node) { + // SCOPED_TRACE(NYsonPull::detail::cescape::quote(data)); auto input_range = NYsonPull::TStreamEventsRange( - NYsonPull::NInput::FromMemory(data), - mode); - for (auto& event : input_range) { - Y_UNUSED(event); - } + NYsonPull::NInput::FromMemory(data), + mode); + for (auto& event : input_range) { + Y_UNUSED(event); + } } #define ACCEPT(data) UNIT_ASSERT_NO_EXCEPTION(consume(data)) @@ -76,7 +76,7 @@ namespace { Y_UNIT_TEST_SUITE(Reader) { Y_UNIT_TEST(ScalarEntity) { test_scalar(TStringBuf("#"), NYsonPull::TScalar{}); - } + } Y_UNIT_TEST(ScalarBoolean) { test_scalar(TStringBuf("%true"), true); @@ -85,13 +85,13 @@ Y_UNIT_TEST_SUITE(Reader) { test_scalar(TStringBuf("\x05"sv), true); test_scalar(TStringBuf("\x04"sv), false); - REJECT("%"); - REJECT("%trueth"); - REJECT("%tru"); - REJECT("%falseth"); - REJECT("%fals"); - REJECT("%hithere"); - } + REJECT("%"); + REJECT("%trueth"); + REJECT("%tru"); + REJECT("%falseth"); + REJECT("%fals"); + REJECT("%hithere"); + } Y_UNIT_TEST(ScalarInt64) { test_scalar(TStringBuf("1"), i64{1}); @@ -107,10 +107,10 @@ Y_UNIT_TEST_SUITE(Reader) { test_scalar(TStringBuf("\x02\xbf\x9a\x0c"sv), i64{-100000}); test_scalar(TStringBuf("\x02\xfe\xff\xff\xff\xff\xff\xff\xff\xff\x01"sv), i64{9223372036854775807}); - REJECT("1a2"); - REJECT("1-1-1-1"); - REJECT("1+0"); - } + REJECT("1a2"); + REJECT("1-1-1-1"); + REJECT("1+0"); + } Y_UNIT_TEST(SclarUInt64) { test_scalar(TStringBuf("1u"), ui64{1}); @@ -122,12 +122,12 @@ Y_UNIT_TEST_SUITE(Reader) { test_scalar(TStringBuf("18446744073709551615u"), ui64{18446744073709551615u}); test_scalar(TStringBuf("+18446744073709551615u"), ui64{18446744073709551615u}); - REJECT("1a2u"); - REJECT("1-1-1-1u"); - REJECT("1+0u"); + REJECT("1a2u"); + REJECT("1-1-1-1u"); + REJECT("1+0u"); - // TODO: binary - } + // TODO: binary + } Y_UNIT_TEST(ScalarFloat64) { test_scalar(TStringBuf("0.0"), double{0.0}); @@ -171,19 +171,19 @@ Y_UNIT_TEST_SUITE(Reader) { TStringBuf("%-inf"), double{-std::numeric_limits<double>::infinity()}); - REJECT("++0.0"); - REJECT("++1.0"); - REJECT("++.1"); - REJECT("1.0.0"); - //REJECT("1e+10000"); + REJECT("++0.0"); + REJECT("++1.0"); + REJECT("++.1"); + REJECT("1.0.0"); + //REJECT("1e+10000"); REJECT(TStringBuf("\x03\x00\x00\x00\x00\x00\x00\x00"sv)); - // XXX: Questionable behaviour? - ACCEPT("+.0"); - ACCEPT("-.0"); - // XXX: Rejected on Mac OS, accepted on Linux (?!) - //REJECT(".0"); - //REJECT(".5"); + // XXX: Questionable behaviour? + ACCEPT("+.0"); + ACCEPT("-.0"); + // XXX: Rejected on Mac OS, accepted on Linux (?!) + //REJECT(".0"); + //REJECT(".5"); REJECT("%NaN"); REJECT("%+inf"); @@ -191,118 +191,118 @@ Y_UNIT_TEST_SUITE(Reader) { REJECT("%na"); REJECT("%in"); REJECT("%-in"); - } + } Y_UNIT_TEST(ScalarString) { test_scalar(TStringBuf(R"(foobar)"), TStringBuf("foobar")); test_scalar(TStringBuf(R"(foobar11)"), TStringBuf("foobar11")); test_scalar(TStringBuf(R"("foobar")"), TStringBuf("foobar")); - // wat? "\x0cf" parsed as a single char? no way! + // wat? "\x0cf" parsed as a single char? no way! test_scalar("\x01\x0c" "foobar"sv, TStringBuf("foobar")); - REJECT(R"("foobar)"); + REJECT(R"("foobar)"); REJECT("\x01\x0c" "fooba"sv); REJECT("\x01\x0d" "foobar"sv); // negative length - } + } Y_UNIT_TEST(EmptyList) { - auto reader = memory_reader("[]", NYsonPull::EStreamType::Node); + auto reader = memory_reader("[]", NYsonPull::EStreamType::Node); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginList, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndList, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); - REJECT("["); - REJECT("]"); - } + REJECT("["); + REJECT("]"); + } Y_UNIT_TEST(EmptyMap) { - auto reader = memory_reader("{}", NYsonPull::EStreamType::Node); + auto reader = memory_reader("{}", NYsonPull::EStreamType::Node); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginMap, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndMap, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); - REJECT("{"); - REJECT("}"); - } + REJECT("{"); + REJECT("}"); + } Y_UNIT_TEST(Sample) { - auto reader = memory_reader( - R"({"11"=11;"nothing"=#;"zero"=0.;"foo"="bar";"list"=[1;2;3]})", - NYsonPull::EStreamType::Node); + auto reader = memory_reader( + R"({"11"=11;"nothing"=#;"zero"=0.;"foo"="bar";"list"=[1;2;3]})", + NYsonPull::EStreamType::Node); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginMap, reader.NextEvent().Type()); - { + { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); UNIT_ASSERT_VALUES_EQUAL(TStringBuf("11"), e.AsString()); - } - { + } + { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{i64{11}}, e.AsScalar()); - } + } - { + { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); UNIT_ASSERT_VALUES_EQUAL(TStringBuf("nothing"), e.AsString()); - } - { + } + { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{}, e.AsScalar()); - } + } - { + { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); UNIT_ASSERT_VALUES_EQUAL(TStringBuf("zero"), e.AsString()); - } - { + } + { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{0.0}, e.AsScalar()); - } + } - { + { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); UNIT_ASSERT_VALUES_EQUAL(TStringBuf("foo"), e.AsString()); - } - { + } + { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{TStringBuf("bar")}, e.AsScalar()); - } + } - { + { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Key, e.Type()); UNIT_ASSERT_VALUES_EQUAL(TStringBuf("list"), e.AsString()); - } + } UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginList, reader.NextEvent().Type()); - { + { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{i64{1}}, e.AsScalar()); - } - { + } + { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{i64{2}}, e.AsScalar()); - } - { + } + { auto& e = reader.NextEvent(); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, e.Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::TScalar{i64{3}}, e.AsScalar()); - } + } UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndList, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndMap, reader.NextEvent().Type()); @@ -310,55 +310,55 @@ Y_UNIT_TEST_SUITE(Reader) { } Y_UNIT_TEST(Accept) { - ACCEPT("[]"); - ACCEPT("{}"); - ACCEPT("<>[]"); - ACCEPT("<>{}"); - ACCEPT("[{};{};{}]"); - ACCEPT("[{};{};{};]"); - ACCEPT("[<>{};<>{};<>{}]"); - ACCEPT("[<>{};<>{};<>{};]"); - - ACCEPT("foo"); - ACCEPT("[foo]"); - ACCEPT("[foo;]"); - ACCEPT("{foo=foo}"); - ACCEPT("{foo=foo;}"); - ACCEPT("<>{foo=foo}"); - ACCEPT("{foo=<foo=foo>foo}"); - ACCEPT("{foo=<foo=foo;>foo}"); - ACCEPT("{foo=<foo=foo>[foo;foo]}"); + ACCEPT("[]"); + ACCEPT("{}"); + ACCEPT("<>[]"); + ACCEPT("<>{}"); + ACCEPT("[{};{};{}]"); + ACCEPT("[{};{};{};]"); + ACCEPT("[<>{};<>{};<>{}]"); + ACCEPT("[<>{};<>{};<>{};]"); + + ACCEPT("foo"); + ACCEPT("[foo]"); + ACCEPT("[foo;]"); + ACCEPT("{foo=foo}"); + ACCEPT("{foo=foo;}"); + ACCEPT("<>{foo=foo}"); + ACCEPT("{foo=<foo=foo>foo}"); + ACCEPT("{foo=<foo=foo;>foo}"); + ACCEPT("{foo=<foo=foo>[foo;foo]}"); } Y_UNIT_TEST(Reject) { - REJECT("["); - REJECT("{"); - REJECT("<"); + REJECT("["); + REJECT("{"); + REJECT("<"); - REJECT("[[}]"); - REJECT("<>{]"); - REJECT("[>]"); + REJECT("[[}]"); + REJECT("<>{]"); + REJECT("[>]"); - REJECT("<><>[]"); - REJECT("[<>;<>]"); + REJECT("<><>[]"); + REJECT("[<>;<>]"); - REJECT("{<>foo=foo}"); - REJECT("{foo=<>}"); - REJECT("{foo}"); + REJECT("{<>foo=foo}"); + REJECT("{foo=<>}"); + REJECT("{foo}"); - REJECT("<a=b>"); - REJECT("<>"); + REJECT("<a=b>"); + REJECT("<>"); REJECT("@"); - } + } Y_UNIT_TEST(ReadPastEnd) { - auto reader = memory_reader("#", NYsonPull::EStreamType::Node); + auto reader = memory_reader("#", NYsonPull::EStreamType::Node); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::BeginStream, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::Scalar, reader.NextEvent().Type()); UNIT_ASSERT_VALUES_EQUAL(NYsonPull::EEventType::EndStream, reader.NextEvent().Type()); UNIT_ASSERT_EXCEPTION(reader.NextEvent(), NYsonPull::NException::TBadInput); - } + } Y_UNIT_TEST(BadInput) { // max_size<ui32> < varint size < max_size<ui64> @@ -405,6 +405,6 @@ Y_UNIT_TEST_SUITE(Reader) { REJECT2("a=[1]; b=foobar", NYsonPull::EStreamType::Node); REJECT2("a=[1]; b=foobar", NYsonPull::EStreamType::ListFragment); ACCEPT2("a=[1]; b=foobar", NYsonPull::EStreamType::MapFragment); - } + } } // Y_UNIT_TEST_SUITE(Reader) diff --git a/library/cpp/yson_pull/ut/writer_ut.cpp b/library/cpp/yson_pull/ut/writer_ut.cpp index 5c304bad0f..4002574fd3 100644 --- a/library/cpp/yson_pull/ut/writer_ut.cpp +++ b/library/cpp/yson_pull/ut/writer_ut.cpp @@ -11,34 +11,34 @@ using namespace std::string_view_literals; namespace { - template <typename Writer, typename Function> + template <typename Writer, typename Function> TString with_writer(Function&& function) { TString result; - auto writer = NYsonPull::NDetail::make_writer<Writer>( + auto writer = NYsonPull::NDetail::make_writer<Writer>( NYsonPull::NOutput::FromString(&result), - NYsonPull::EStreamType::Node); + NYsonPull::EStreamType::Node); - function(writer); + function(writer); return result; - } + } - template <typename Writer> + template <typename Writer> TString to_yson_string(const NYsonPull::TScalar& value) { - return with_writer<Writer>([&](NYsonPull::TWriter& writer) { + return with_writer<Writer>([&](NYsonPull::TWriter& writer) { writer.BeginStream().Scalar(value).EndStream(); - }); - } + }); + } - template <typename T> + template <typename T> TString to_yson_binary_string(T&& value) { return to_yson_string<NYsonPull::NDetail::TBinaryWriterImpl>(std::forward<T>(value)); - } + } - template <typename T> + template <typename T> TString to_yson_text_string(T&& value) { return to_yson_string<NYsonPull::NDetail::TTextWriterImpl>(std::forward<T>(value)); - } + } } // anonymous namespace @@ -46,80 +46,80 @@ namespace { Y_UNIT_TEST_SUITE(Writer) { Y_UNIT_TEST(TextEntity) { - UNIT_ASSERT_VALUES_EQUAL( - "#", - to_yson_text_string(NYsonPull::TScalar{})); - } + UNIT_ASSERT_VALUES_EQUAL( + "#", + to_yson_text_string(NYsonPull::TScalar{})); + } Y_UNIT_TEST(TextBoolean) { - UNIT_ASSERT_VALUES_EQUAL( - "%false", - to_yson_text_string(NYsonPull::TScalar{false})); - UNIT_ASSERT_VALUES_EQUAL( - "%true", - to_yson_text_string(NYsonPull::TScalar{true})); - } + UNIT_ASSERT_VALUES_EQUAL( + "%false", + to_yson_text_string(NYsonPull::TScalar{false})); + UNIT_ASSERT_VALUES_EQUAL( + "%true", + to_yson_text_string(NYsonPull::TScalar{true})); + } Y_UNIT_TEST(TextInt64) { - UNIT_ASSERT_VALUES_EQUAL( - "0", - to_yson_text_string(NYsonPull::TScalar{i64{0}})); - UNIT_ASSERT_VALUES_EQUAL( - "200", - to_yson_text_string(NYsonPull::TScalar{i64{200}})); - UNIT_ASSERT_VALUES_EQUAL( - "20000", - to_yson_text_string(NYsonPull::TScalar{i64{20000}})); - UNIT_ASSERT_VALUES_EQUAL( - "200000000", - to_yson_text_string(NYsonPull::TScalar{i64{200000000}})); - UNIT_ASSERT_VALUES_EQUAL( - "20000000000000000", - to_yson_text_string(NYsonPull::TScalar{i64{20000000000000000}})); - UNIT_ASSERT_VALUES_EQUAL( - "9223372036854775807", - to_yson_text_string(NYsonPull::TScalar{i64{INT64_MAX}})); + UNIT_ASSERT_VALUES_EQUAL( + "0", + to_yson_text_string(NYsonPull::TScalar{i64{0}})); + UNIT_ASSERT_VALUES_EQUAL( + "200", + to_yson_text_string(NYsonPull::TScalar{i64{200}})); + UNIT_ASSERT_VALUES_EQUAL( + "20000", + to_yson_text_string(NYsonPull::TScalar{i64{20000}})); + UNIT_ASSERT_VALUES_EQUAL( + "200000000", + to_yson_text_string(NYsonPull::TScalar{i64{200000000}})); + UNIT_ASSERT_VALUES_EQUAL( + "20000000000000000", + to_yson_text_string(NYsonPull::TScalar{i64{20000000000000000}})); + UNIT_ASSERT_VALUES_EQUAL( + "9223372036854775807", + to_yson_text_string(NYsonPull::TScalar{i64{INT64_MAX}})); - UNIT_ASSERT_VALUES_EQUAL( - "-200", - to_yson_text_string(NYsonPull::TScalar{i64{-200}})); - UNIT_ASSERT_VALUES_EQUAL( - "-20000", - to_yson_text_string(NYsonPull::TScalar{i64{-20000}})); - UNIT_ASSERT_VALUES_EQUAL( - "-200000000", - to_yson_text_string(NYsonPull::TScalar{i64{-200000000}})); - UNIT_ASSERT_VALUES_EQUAL( - "-20000000000000000", - to_yson_text_string(NYsonPull::TScalar{i64{-20000000000000000}})); - UNIT_ASSERT_VALUES_EQUAL( - "-9223372036854775808", - to_yson_text_string(NYsonPull::TScalar{i64{INT64_MIN}})); - } + UNIT_ASSERT_VALUES_EQUAL( + "-200", + to_yson_text_string(NYsonPull::TScalar{i64{-200}})); + UNIT_ASSERT_VALUES_EQUAL( + "-20000", + to_yson_text_string(NYsonPull::TScalar{i64{-20000}})); + UNIT_ASSERT_VALUES_EQUAL( + "-200000000", + to_yson_text_string(NYsonPull::TScalar{i64{-200000000}})); + UNIT_ASSERT_VALUES_EQUAL( + "-20000000000000000", + to_yson_text_string(NYsonPull::TScalar{i64{-20000000000000000}})); + UNIT_ASSERT_VALUES_EQUAL( + "-9223372036854775808", + to_yson_text_string(NYsonPull::TScalar{i64{INT64_MIN}})); + } Y_UNIT_TEST(TextUInt64) { - UNIT_ASSERT_VALUES_EQUAL( - "0u", - to_yson_text_string(NYsonPull::TScalar{ui64{0}})); - UNIT_ASSERT_VALUES_EQUAL( - "200u", - to_yson_text_string(NYsonPull::TScalar{ui64{200}})); - UNIT_ASSERT_VALUES_EQUAL( - "20000u", - to_yson_text_string(NYsonPull::TScalar{ui64{20000}})); - UNIT_ASSERT_VALUES_EQUAL( - "200000000u", - to_yson_text_string(NYsonPull::TScalar{ui64{200000000}})); - UNIT_ASSERT_VALUES_EQUAL( - "20000000000000000u", - to_yson_text_string(NYsonPull::TScalar{ui64{20000000000000000}})); - UNIT_ASSERT_VALUES_EQUAL( - "9223372036854775807u", - to_yson_text_string(NYsonPull::TScalar{ui64{INT64_MAX}})); - UNIT_ASSERT_VALUES_EQUAL( - "18446744073709551615u", - to_yson_text_string(NYsonPull::TScalar{ui64{UINT64_MAX}})); - } + UNIT_ASSERT_VALUES_EQUAL( + "0u", + to_yson_text_string(NYsonPull::TScalar{ui64{0}})); + UNIT_ASSERT_VALUES_EQUAL( + "200u", + to_yson_text_string(NYsonPull::TScalar{ui64{200}})); + UNIT_ASSERT_VALUES_EQUAL( + "20000u", + to_yson_text_string(NYsonPull::TScalar{ui64{20000}})); + UNIT_ASSERT_VALUES_EQUAL( + "200000000u", + to_yson_text_string(NYsonPull::TScalar{ui64{200000000}})); + UNIT_ASSERT_VALUES_EQUAL( + "20000000000000000u", + to_yson_text_string(NYsonPull::TScalar{ui64{20000000000000000}})); + UNIT_ASSERT_VALUES_EQUAL( + "9223372036854775807u", + to_yson_text_string(NYsonPull::TScalar{ui64{INT64_MAX}})); + UNIT_ASSERT_VALUES_EQUAL( + "18446744073709551615u", + to_yson_text_string(NYsonPull::TScalar{ui64{UINT64_MAX}})); + } Y_UNIT_TEST(TextFloat64) { UNIT_ASSERT_VALUES_EQUAL( @@ -131,97 +131,97 @@ Y_UNIT_TEST_SUITE(Writer) { UNIT_ASSERT_VALUES_EQUAL( "%nan", to_yson_text_string(NYsonPull::TScalar{std::numeric_limits<double>::quiet_NaN()})); - } + } Y_UNIT_TEST(TextString) { - UNIT_ASSERT_VALUES_EQUAL( - R"("")", - to_yson_text_string(NYsonPull::TScalar{""})); - UNIT_ASSERT_VALUES_EQUAL( - R"("hello")", - to_yson_text_string(NYsonPull::TScalar{"hello"})); - UNIT_ASSERT_VALUES_EQUAL( - R"("hello\nworld")", - to_yson_text_string(NYsonPull::TScalar{"hello\nworld"})); - } + UNIT_ASSERT_VALUES_EQUAL( + R"("")", + to_yson_text_string(NYsonPull::TScalar{""})); + UNIT_ASSERT_VALUES_EQUAL( + R"("hello")", + to_yson_text_string(NYsonPull::TScalar{"hello"})); + UNIT_ASSERT_VALUES_EQUAL( + R"("hello\nworld")", + to_yson_text_string(NYsonPull::TScalar{"hello\nworld"})); + } - // =================== Binary format ===================== + // =================== Binary format ===================== Y_UNIT_TEST(BinaryEntity) { - UNIT_ASSERT_VALUES_EQUAL( - "#", - to_yson_binary_string(NYsonPull::TScalar{})); - } + UNIT_ASSERT_VALUES_EQUAL( + "#", + to_yson_binary_string(NYsonPull::TScalar{})); + } Y_UNIT_TEST(BinaryBoolean) { - UNIT_ASSERT_VALUES_EQUAL( + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x4"), - to_yson_binary_string(NYsonPull::TScalar{false})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{false})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x5"), - to_yson_binary_string(NYsonPull::TScalar{true})); - } + to_yson_binary_string(NYsonPull::TScalar{true})); + } Y_UNIT_TEST(BinaryInt64) { - UNIT_ASSERT_VALUES_EQUAL( + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x2\0"sv), - to_yson_binary_string(NYsonPull::TScalar{i64{0}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{i64{0}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x2\x90\x3"), - to_yson_binary_string(NYsonPull::TScalar{i64{200}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{i64{200}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x2\xC0\xB8\x2"), - to_yson_binary_string(NYsonPull::TScalar{i64{20000}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{i64{20000}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x2\x80\x88\xDE\xBE\x1"), - to_yson_binary_string(NYsonPull::TScalar{i64{200000000}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{i64{200000000}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x2\x80\x80\x90\xF8\x9B\xF9\x86G"), - to_yson_binary_string(NYsonPull::TScalar{i64{20000000000000000}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{i64{20000000000000000}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x2\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x1"), - to_yson_binary_string(NYsonPull::TScalar{i64{INT64_MAX}})); + to_yson_binary_string(NYsonPull::TScalar{i64{INT64_MAX}})); - UNIT_ASSERT_VALUES_EQUAL( + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x2\x8F\x3"), - to_yson_binary_string(NYsonPull::TScalar{i64{-200}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{i64{-200}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x2\xBF\xB8\x2"), - to_yson_binary_string(NYsonPull::TScalar{i64{-20000}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{i64{-20000}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x2\xFF\x87\xDE\xBE\x1"), - to_yson_binary_string(NYsonPull::TScalar{i64{-200000000}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{i64{-200000000}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x2\xFF\xFF\x8F\xF8\x9B\xF9\x86G"), - to_yson_binary_string(NYsonPull::TScalar{i64{-20000000000000000}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{i64{-20000000000000000}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x2\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x1"), - to_yson_binary_string(NYsonPull::TScalar{i64{INT64_MIN}})); - } + to_yson_binary_string(NYsonPull::TScalar{i64{INT64_MIN}})); + } Y_UNIT_TEST(BinaryUInt64) { - UNIT_ASSERT_VALUES_EQUAL( + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x6\0"sv), - to_yson_binary_string(NYsonPull::TScalar{ui64{0}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{ui64{0}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x6\xC8\x1"), - to_yson_binary_string(NYsonPull::TScalar{ui64{200}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{ui64{200}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x6\xA0\x9C\x1"), - to_yson_binary_string(NYsonPull::TScalar{ui64{20000}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{ui64{20000}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x6\x80\x84\xAF_"), - to_yson_binary_string(NYsonPull::TScalar{ui64{200000000}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{ui64{200000000}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x6\x80\x80\x88\xFC\xCD\xBC\xC3#"), - to_yson_binary_string(NYsonPull::TScalar{ui64{20000000000000000}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{ui64{20000000000000000}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x6\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F"), - to_yson_binary_string(NYsonPull::TScalar{ui64{INT64_MAX}})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{ui64{INT64_MAX}})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x6\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x1"), - to_yson_binary_string(NYsonPull::TScalar{ui64{UINT64_MAX}})); - } + to_yson_binary_string(NYsonPull::TScalar{ui64{UINT64_MAX}})); + } Y_UNIT_TEST(BinaryFloat64) { UNIT_ASSERT_VALUES_EQUAL( @@ -239,18 +239,18 @@ Y_UNIT_TEST_SUITE(Writer) { UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x03\x9a\x99\x99\x99\x99\x99\xf1\xbf"), to_yson_binary_string(NYsonPull::TScalar{double{-1.1}})); - } + } Y_UNIT_TEST(BinaryString) { - UNIT_ASSERT_VALUES_EQUAL( + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x1\0"sv), - to_yson_binary_string(NYsonPull::TScalar{""})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{""})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x1\nhello"), - to_yson_binary_string(NYsonPull::TScalar{"hello"})); - UNIT_ASSERT_VALUES_EQUAL( + to_yson_binary_string(NYsonPull::TScalar{"hello"})); + UNIT_ASSERT_VALUES_EQUAL( TStringBuf("\x1\x16hello\nworld"), - to_yson_binary_string(NYsonPull::TScalar{"hello\nworld"})); - } + to_yson_binary_string(NYsonPull::TScalar{"hello\nworld"})); + } } // Y_UNIT_TEST_SUITE(Writer) diff --git a/library/cpp/yson_pull/writer.cpp b/library/cpp/yson_pull/writer.cpp index 1df92bf40f..94b93d1958 100644 --- a/library/cpp/yson_pull/writer.cpp +++ b/library/cpp/yson_pull/writer.cpp @@ -5,26 +5,26 @@ using namespace NYsonPull; TWriter NYsonPull::MakeBinaryWriter( THolder<NOutput::IStream> stream, - EStreamType mode) { + EStreamType mode) { return NYsonPull::NDetail::make_writer<NYsonPull::NDetail::TBinaryWriterImpl>( std::move(stream), - mode); + mode); } TWriter NYsonPull::MakeTextWriter( THolder<NOutput::IStream> stream, - EStreamType mode) { + EStreamType mode) { return NYsonPull::NDetail::make_writer<NYsonPull::NDetail::TTextWriterImpl>( std::move(stream), - mode); + mode); } TWriter NYsonPull::MakePrettyTextWriter( THolder<NOutput::IStream> stream, EStreamType mode, - size_t indent_size) { + size_t indent_size) { return NYsonPull::NDetail::make_writer<NYsonPull::NDetail::TPrettyWriterImpl>( std::move(stream), mode, - indent_size); + indent_size); } diff --git a/library/cpp/yson_pull/writer.h b/library/cpp/yson_pull/writer.h index dec63328be..df11084e28 100644 --- a/library/cpp/yson_pull/writer.h +++ b/library/cpp/yson_pull/writer.h @@ -8,119 +8,119 @@ #include <memory> namespace NYsonPull { - //! \brief YSON writer facade class - //! - //! Owns a YSON consumer and a corresponding output stream. - //! Methods invoke corresponding \p NYsonPull::IConsumer methods and can be chained. - class TWriter { + //! \brief YSON writer facade class + //! + //! Owns a YSON consumer and a corresponding output stream. + //! Methods invoke corresponding \p NYsonPull::IConsumer methods and can be chained. + class TWriter { THolder<NOutput::IStream> Stream_; THolder<IConsumer> Impl_; - public: - TWriter( - THolder<NOutput::IStream> stream, - THolder<IConsumer> impl) + public: + TWriter( + THolder<NOutput::IStream> stream, + THolder<IConsumer> impl) : Stream_{std::move(stream)} , Impl_{std::move(impl)} { - } + } - //! \brief Get a reference to underlying consumer. - //! - //! Useful with \p NYsonPull::bridge + //! \brief Get a reference to underlying consumer. + //! + //! Useful with \p NYsonPull::bridge IConsumer& GetConsumer() { return *Impl_; - } + } TWriter& BeginStream() { Impl_->OnBeginStream(); - return *this; - } + return *this; + } TWriter& EndStream() { Impl_->OnEndStream(); - return *this; - } + return *this; + } TWriter& BeginList() { Impl_->OnBeginList(); - return *this; - } + return *this; + } TWriter& EndList() { Impl_->OnEndList(); - return *this; - } + return *this; + } TWriter& BeginMap() { Impl_->OnBeginMap(); - return *this; - } + return *this; + } TWriter& EndMap() { Impl_->OnEndMap(); - return *this; - } + return *this; + } TWriter& BeginAttributes() { Impl_->OnBeginAttributes(); - return *this; - } + return *this; + } TWriter& EndAttributes() { Impl_->OnEndAttributes(); - return *this; - } + return *this; + } TWriter& Key(TStringBuf name) { Impl_->OnKey(name); - return *this; - } + return *this; + } TWriter& Entity() { Impl_->OnEntity(); - return *this; - } + return *this; + } TWriter& Boolean(bool value) { Impl_->OnScalarBoolean(value); - return *this; - } + return *this; + } TWriter& Int64(i64 value) { Impl_->OnScalarInt64(value); - return *this; - } + return *this; + } TWriter& UInt64(ui64 value) { Impl_->OnScalarUInt64(value); - return *this; - } + return *this; + } TWriter& Float64(double value) { Impl_->OnScalarFloat64(value); - return *this; - } + return *this; + } TWriter& String(TStringBuf value) { Impl_->OnScalarString(value); - return *this; - } + return *this; + } TWriter& Scalar(const TScalar& value) { Impl_->OnScalar(value); - return *this; - } + return *this; + } TWriter& Event(const TEvent& value) { Impl_->OnEvent(value); - return *this; - } - }; + return *this; + } + }; - //! \brief Construct a writer for binary YSON format. + //! \brief Construct a writer for binary YSON format. TWriter MakeBinaryWriter( - THolder<NOutput::IStream> stream, - EStreamType mode); + THolder<NOutput::IStream> stream, + EStreamType mode); - //! \brief Construct a writer for text YSON format. + //! \brief Construct a writer for text YSON format. TWriter MakeTextWriter( - THolder<NOutput::IStream> stream, - EStreamType mode); + THolder<NOutput::IStream> stream, + EStreamType mode); - //! \brief Construct a writer for pretty text YSON format. + //! \brief Construct a writer for pretty text YSON format. TWriter MakePrettyTextWriter( - THolder<NOutput::IStream> stream, - EStreamType mode, - size_t indent_size = 4); + THolder<NOutput::IStream> stream, + EStreamType mode, + size_t indent_size = 4); -} +} |